{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from utils.input_pipeline import get_image_folders\n",
    "from utils.diagnostic import predict, top_k_accuracy, per_class_accuracy,\\\n",
    "    entropy, model_calibration, most_confused_classes, most_inaccurate_k_classes\n",
    "    \n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.backends.cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from get_squeezenet import get_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model, _, _ = get_model()\n",
    "model.load_state_dict(torch.load('model.pytorch_state'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Error analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get all predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "_, val_folder = get_image_folders()\n",
    "\n",
    "val_iterator_no_shuffle = DataLoader(\n",
    "    val_folder, batch_size=256, shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40/40 [00:13<00:00,  3.26it/s]\n"
     ]
    }
   ],
   "source": [
    "val_predictions, val_true_targets = predict(model, val_iterator_no_shuffle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get human readable class names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = '/home/ubuntu/data/tiny-imagenet-200/'\n",
    "\n",
    "class_names = pd.read_csv(data_dir + 'words.txt', sep='\\t', header=None)\n",
    "class_names.columns = ['code', 'description']\n",
    "\n",
    "decode = {}\n",
    "for i, row in class_names.iterrows():\n",
    "    decode[row.code] = row.description\n",
    "    \n",
    "\n",
    "# class index to human readable format\n",
    "decode = {\n",
    "    val_folder.class_to_idx[code]: decode[code] \n",
    "    for code in val_folder.class_to_idx\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### number of misclassified images (there are overall 10k images in the val dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4755"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits = val_predictions.argmax(1) == val_true_targets\n",
    "n_errors = (~hits).sum()\n",
    "n_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### logloss and accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.968486776923378"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(val_true_targets, val_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.52449999999999997"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(val_true_targets, val_predictions.argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.64329999999999998, 0.70879999999999999, 0.74480000000000002, 0.77380000000000004, 0.85340000000000005]\n"
     ]
    }
   ],
   "source": [
    "print(top_k_accuracy(val_true_targets, val_predictions, k=(2, 3, 4, 5, 10)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### entropy of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHMBJREFUeJzt3Xt0VdW59/HvQ4AGkNIisUe5HPC8KCIkAYIoKFdFBAro\ngFdRuXmhdIh62iGI+laRY60dctQjWCMiRnu0IIpKGVQUlQIDKIk1oiAgKpUoBxAVLxS5+Lx/7EXO\nJoRkJdnJzl75fcZgkLXW3Gs92cn47Zm55p7b3B0REYmWeskuQEREEk/hLiISQQp3EZEIUriLiESQ\nwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCKofrIu3KJFC2/btm2yLi8ikpLeeuutz909o7x2SQv3\ntm3bUlBQkKzLi4ikJDP7R5h2GpYREYkghbuISAQp3EVEIihpY+4iUjGHDh2iqKiIAwcOJLsUqQHp\n6em0atWKBg0aVOrxCneRFFFUVETTpk1p27YtZpbscqQauTt79+6lqKiIdu3aVeocGpYRSREHDhzg\n5JNPVrDXAWbGySefXKW/0hTuIilEwV53VPVnrXAXEYkgjbmLpKhr8/ITer4nxndP6PkSLS8vj4ED\nB3LaaadV+7XGjx/P0KFDGTlyJNdddx2//vWv6dixY6ltV6xYQcOGDenZsycAubm5NG7cmLFjx1Z7\nnWUpN9zNbB4wFNjt7p1KOW7AfwGDgf3AeHf/e6ILjRf2l7q2/7KK1BWHDx+mfv36J9wOIy8vj06d\nOlU63CtzTYC5c+eWeXzFihWcdNJJxeE+adKkStWXaGGGZfKAQWUcvwRoH/ybCDxa9bJEpDZ6+umn\nyczMJCsrizFjxgCwfft2+vfvT2ZmJgMGDOCTTz4BYr3fSZMm0aNHD6ZOncr06dMZM2YMvXr1YsyY\nMRw5coQpU6bQvXt3MjMzeeyxx4qv8/vf/57OnTuTlZXFtGnTeP755ykoKOCqq64iOzubf/7zn8fU\n1bdvX26++Ways7Pp1KkT69evBwh9TXdn8uTJnHnmmVx44YXs3r37mHMfXSrllVdeoWvXrmRlZTFg\nwAC2b99Obm4uDz74INnZ2axatYrp06czc+ZMAAoLCzn33HPJzMzk0ksv5csvvyw+56233so555zD\nGWecwapVqxL+syr3ZczdV5pZ2zKaDAeedncH1pnZT8zsVHffmaAaRaQW2LhxI/fccw9r1qyhRYsW\nfPHFFwDceOONjBs3jnHjxjFv3jxuuukmXnrpJSA2fXPNmjWkpaUxffp0Nm3axOrVq2nUqBFz5syh\nWbNm5Ofn8/3339OrVy8GDhzI5s2befnll/nb3/5G48aN+eKLL2jevDmzZ89m5syZ5OTklFrf/v37\nKSwsZOXKlVxzzTW89957AKGu+fbbb7NlyxY2bdrErl276NixI9dcc80x59+zZw/XX389K1eupF27\ndsV1TZo0iZNOOolbbrkFgNdff734MWPHjmXWrFn06dOHO++8k7vvvpuHHnoIiP0lsX79epYuXcrd\nd9/N8uXLE/rzSsSYe0tgR9x2UbDvuHA3s4nEeve0adMmAZcWkZryxhtvMGrUKFq0aAFA8+bNAVi7\ndi2LFi0CYMyYMUydOrX4MaNGjSItLa14e9iwYTRq1AiAV199lQ0bNvD8888DsG/fPj744AOWL1/O\nhAkTaNy48THXKc/o0aMB6N27N19//TVfffVV6GuuXLmS0aNHk5aWxmmnnUb//v2PO/+6devo3bt3\n8bzz8urat28fX331FX369AFg3LhxjBo1qvj4ZZddBkC3bt3Yvn17qO+xImr0hqq7zwHmAOTk5HhN\nXltEal6TJk1OuO3uzJo1i4svvviYNsuWLavUtUpOHTy6HeaaS5curdQ1q+JHP/oRAGlpaRw+fDjh\n50/EVMhPgdZx262CfSISIf3792fhwoXs3bsXoHhYpmfPnsyfPx+AZ555hgsuuCDU+S6++GIeffRR\nDh06BMDWrVv57rvvuOiii3jyySfZv3//Mddp2rQp33zzzQnPt2DBAgBWr15Ns2bNaNasWehr9u7d\nmwULFnDkyBF27tzJm2++edxjzz33XFauXMnHH38cqq5mzZrx05/+tHg8/Y9//GNxL74mJKLnvhiY\nbGbzgR7APo23i1S/mp4NdvbZZ3PHHXfQp08f0tLS6NKlC3l5ecyaNYsJEyZw//33k5GRwZNPPhnq\nfNdddx3bt2+na9euuDsZGRm89NJLDBo0iMLCQnJycmjYsCGDBw/m3nvvLb5B26hRI9auXVs81HJU\neno6Xbp04dChQ8ybN69C17z00kt544036NixI23atOG888477rEZGRnMmTOHyy67jB9++IFTTjmF\n1157jZ///OeMHDmSl19+mVmzZh3zmKeeeopJkyaxf/9+Tj/99NDPTSJY7D5oGQ3M/gT0BVoAu4C7\ngAYA7p4bTIWcTWxGzX5ggruX+ykcOTk5XtkP69BUSKmL3n//fc4666xkl1Er9e3bt8ybramqtJ+5\nmb3l7uV+o2Fmy4wu57gDN5R3HhERqTl6h6qIpLwVK1Yku4RaR2vLiIhEkMJdRCSCFO4iIhGkcBcR\niSDdUBVJVW/+LrHn63dbQk6zePFiNm3axLRp0yr0uEROZywoKODpp5/m4Ycf5vvvv2fIkCF8/vnn\n3Hbbbbz22mtlLuF7IoWFhXz22WcMHjwYqPz3WVMU7iKSUMOGDWPYsGFJrSEnJ6f4ReLtt98GYuEM\ncPnll1fqnIWFhRQUFBSHe234PsuiYRkRCW379u106NCB8ePHc8YZZ3DVVVexfPlyevXqRfv27Vm/\nfj15eXlMnjwZgIULF9KpUyeysrLo3bs3AEeOHOGWW26hU6dOZGZmHveuToBf/vKX5OTkcPbZZ3PX\nXXcV7582bRodO3YkMzOzeBXG0q6xYsUKhg4dyu7du7n66qvJz88nOzubDz/8sMwlfAHWr1/Peeed\nR5cuXejZsydbtmzh4MGD3HnnnSxYsIDs7GwWLFhwzPdZ1rLHN910Ez179uT0008vXrCsJqjnLiIV\nsm3bNhYuXMi8efPo3r07zz77LKtXr2bx4sXce++9jBgxorjtjBkzWLZsGS1btixepXHOnDls376d\nwsJC6tevX7xGS7zf/va3NG/enCNHjjBgwAA2bNhAy5YtefHFF9m8eTNmVny+0q5x1CmnnMLcuXOZ\nOXMmS5YsOeZYaUv4AnTo0IFVq1ZRv359li9fzu23384LL7zAjBkzKCgoYPbs2UDsw0OOKmvZ4507\nd7J69Wo2b97MsGHDGDlyZBV/AuGo5y4iFdKuXTs6d+5MvXr1OPvssxkwYABmRufOnY9burZXr16M\nHz+exx9/nCNHjgCwfPlyfvGLXxR/KlJpS+c+99xzdO3alS5durBx40Y2bdpEs2bNSE9P59prr2XR\nokXFSwKXdo0wTrSE7759+xg1ahSdOnXiV7/6FRs3biz3XGvXruXKK68EYsser169uvjYiBEjqFev\nHh07dmTXrl2h66sqhbuIVMjRpWoB6tWrV7xdr16945auzc3N5Z577mHHjh1069ateEXJsnz88cfM\nnDmT119/nQ0bNjBkyBAOHDhA/fr1Wb9+PSNHjmTJkiUMGjSo0tcoy29+8xv69evHe++9x5///GcO\nHDhQpfPFP1/lreWVSAp3Eak2H374IT169GDGjBlkZGSwY8cOLrroIh577LHiF4KSwzJff/01TZo0\noVmzZuzatYu//OUvAHz77bfs27ePwYMH8+CDD/LOO++c8BphnGgJ33379tGyZUvg2KGXspYcruyy\nx9VJY+4iqSpBUxer05QpU/jggw9wdwYMGEBWVhadOnVi69atZGZm0qBBA66//vriG5MAWVlZdOnS\nhQ4dOtC6dWt69eoFwDfffMPw4cM5cOAA7s4DDzxwwmv89a9/Lbe2Ey3hO3XqVMaNG8c999zDkCFD\nitv369eP++67j+zsbG677djnvrLLHlencpf8rS5a8lekYrTkb91TlSV/NSwjIhJBCncRkQhSuIuk\nkGQNo0rNq+rPWuEukiLS09PZu3evAr4OcHf27t1Lenp6pc+h2TIiKaJVq1YUFRWxZ8+eZJciNSA9\nPZ1WrVpV+vEKd5EU0aBBg+J3U4qUR8MyIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriL\niESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEhQp3MxtkZlvMbJuZTSvleDMz+7OZvWNmG81sQuJLFRGR\nsMoNdzNLAx4BLgE6AqPNrGOJZjcAm9w9C+gL/KeZNUxwrSIiElKYnvs5wDZ3/8jdDwLzgeEl2jjQ\n1MwMOAn4Ajic0EpFRCS0MOHeEtgRt10U7Is3GzgL+Ax4F7jZ3X9ISIUiIlJhibqhejFQCJwGZAOz\nzezHJRuZ2UQzKzCzAn3ggIhI9QkT7p8CreO2WwX74k0AFnnMNuBjoEPJE7n7HHfPcfecjIyMytYs\nIiLlCBPu+UB7M2sX3CS9Alhcos0nwAAAM/sZcCbwUSILFRGR8Mr9mD13P2xmk4FlQBowz903mtmk\n4Hgu8B9Anpm9Cxhwq7t/Xo11i4hIGUJ9hqq7LwWWltiXG/f1Z8DAxJYmIiKVpXeoiohEkMJdRCSC\nFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuI\nSAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEK\ndxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQgKFe5mNsjMtpjZ\nNjObdoI2fc2s0Mw2mtlfE1umiIhURP3yGphZGvAIcBFQBOSb2WJ33xTX5ifAH4BB7v6JmZ1SXQWL\niEj5wvTczwG2uftH7n4QmA8ML9HmSmCRu38C4O67E1umiIhURJhwbwnsiNsuCvbFOwP4qZmtMLO3\nzGxsogoUEZGKK3dYpgLn6QYMABoBa81snbtvjW9kZhOBiQBt2rRJ0KVFRKSkMD33T4HWcdutgn3x\nioBl7v6du38OrASySp7I3ee4e46752RkZFS2ZhERKUeYcM8H2ptZOzNrCFwBLC7R5mXgfDOrb2aN\ngR7A+4ktVUREwip3WMbdD5vZZGAZkAbMc/eNZjYpOJ7r7u+b2SvABuAHYK67v1edhYuIyImFGnN3\n96XA0hL7ckts3w/cn7jSRESksvQOVRGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQi\nSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriL\niESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGk\ncBcRiSCFu4hIBCncRUQiSOEuIhJBocLdzAaZ2RYz22Zm08po193MDpvZyMSVKCIiFVVuuJtZGvAI\ncAnQERhtZh1P0O73wKuJLlJERComTM/9HGCbu3/k7geB+cDwUtrdCLwA7E5gfSIiUglhwr0lsCNu\nuyjYV8zMWgKXAo+WdSIzm2hmBWZWsGfPnorWKiIiISXqhupDwK3u/kNZjdx9jrvnuHtORkZGgi4t\nIiIl1Q/R5lOgddx2q2BfvBxgvpkBtAAGm9lhd38pIVWKiEiFhAn3fKC9mbUjFupXAFfGN3D3dke/\nNrM8YImCXUQkecoNd3c/bGaTgWVAGjDP3Tea2aTgeG411ygiIhUUpueOuy8FlpbYV2qou/v4qpcl\nIiJVoXeoiohEUKiee6q6Ni+/3DZPjO9eA5WIiNQs9dxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSC\nFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBkV5+IAwtUSAiUaSeu4hIBCncRUQiSOEuIhJBCncR\nkQhSuIuIRJDCXUQkgur8VEiRGvHm78K163db9dYhdYbCPYQwc+FB8+ElAfQiIAmiYRkRkQhSz10k\nysL+JQD6ayBi1HMXEYkg9dwTSOvU1EEV6RmL1CCFu0gq0ouKlEPhLqlP48oix1G4i0iMpmFGisK9\nhmnOfIrQsIekOIW71C0KbakjFO61lGbeSK2l4ZuUEGqeu5kNMrMtZrbNzKaVcvwqM9tgZu+a2Roz\ny0p8qSIiEla5PXczSwMeAS4CioB8M1vs7pvimn0M9HH3L83sEmAO0KM6Cpb/lbK9ew2NiFS7MD33\nc4Bt7v6Rux8E5gPD4xu4+xp3/zLYXAe0SmyZIiJSEWHCvSWwI267KNh3ItcCf6lKUSIiUjUJvaFq\nZv2Ihfv5Jzg+EZgI0KZNm0ReWkRE4oQJ90+B1nHbrYJ9xzCzTGAucIm77y3tRO4+h9h4PDk5OV7h\naqXCNK9epG4KMyyTD7Q3s3Zm1hC4Algc38DM2gCLgDHuvjXxZYqISEWU23N398NmNhlYBqQB89x9\no5lNCo7nAncCJwN/MDOAw+6eU31li0itp/nwSRVqzN3dlwJLS+zLjfv6OuC6xJYmNSkh0yo1xVGk\n1tCHdYiIRJCWH5CEernwuHvtpRqeXdZsWhGpKoW7lGvYl08B8PKDTyW5EhEJS+Fehx0NbRGJHoV7\nBKVCaIcZvtHQjUjlKdxFJLk0ZbJaaLaMiEgEKdxFRCJIwzIpIhXG0RNN4/Iiladwl5SmefUipVO4\niwTCvlCEoRcTSTaFu9QJiQxukVSgG6oiIhGkcBcRiSANyyRZXZwFIyLVT+EuUg00jVOSTeEukiSa\nxinVSeFeTTTcIiLJpBuqIiIRpJ67SC2n8XupDPXcRUQiSD13kQhQ715KUriLSGrQh3pUiMK9gjQL\nRkRSgcJdpI6oM/Pq1cMHFO7F1CMXkSjRbBkRkQhSz11EjpGote9TfngnxSncRaRuCjs2Dyk5Pq9w\nF5FqUWdu4NZSGnMXEYmgyPfcNQtGROqiUOFuZoOA/wLSgLnufl+J4xYcHwzsB8a7+98TXKuISHKk\n4Nz5csPdzNKAR4CLgCIg38wWu/umuGaXAO2Dfz2AR4P/q4165CLRkKjZOWHVlTH+MD33c4Bt7v4R\ngJnNB4YD8eE+HHja3R1YZ2Y/MbNT3X1nwisWEamtalEPP0y4twR2xG0XcXyvvLQ2LQGFu4jUKnVl\nBc0avaFqZhOBicHmt2a2pZKnajEPPk9QWTWtBaq9pqVq3aDak6EG6r69Kg/+1zCNwoT7p0DruO1W\nwb6KtsHd5wBzwhRWFjMrcPecqp4nGVR7zUvVukG1J0Oq1l1SmHnu+UB7M2tnZg2BK4DFJdosBsZa\nzLnAPo23i4gkT7k9d3c/bGaTgWXEpkLOc/eNZjYpOJ4LLCU2DXIbsamQE6qvZBERKU+oMXd3X0os\nwOP35cZ97cANiS2tTFUe2kki1V7zUrVuUO3JkKp1H8NiuSwiIlGitWVERCIo5cLdzAaZ2RYz22Zm\n05JdT1hmNs/MdpvZe8mupSLMrLWZvWlmm8xso5ndnOyawjKzdDNbb2bvBLXfneyaKsLM0szsbTNb\nkuxaKsLMtpvZu2ZWaGYFya6nIoI3YD5vZpvN7H0zOy/ZNVVWSg3LBEshbCVuKQRgdImlEGolM+sN\nfEvsnbydkl1PWGZ2KnCqu//dzJoCbwEjUuQ5N6CJu39rZg2A1cDN7r4uyaWFYma/BnKAH7v70GTX\nE5aZbQdy3D3l5rib2VPAKnefG8wObOzuXyW7rspItZ578VII7n4QOLoUQq3n7iuBL5JdR0W5+86j\ni8C5+zfA+8TefVzrecy3wWaD4F9K9GbMrBUwBJib7FrqCjNrBvQGngBw94OpGuyQeuF+omUOpAaY\nWVugC/C35FYSXjC0UQjsBl5z91Sp/SFgKvBDsgupBAeWm9lbwbvSU0U7YA/wZDAcNtfMmiS7qMpK\ntXCXJDGzk4AXgH9396+TXU9Y7n7E3bOJvWv6HDOr9UNiZjYU2O3ubyW7lko6P3jOLwFuCIYkU0F9\noCvwqLt3Ab4DUua+XkmpFu6hljmQxArGq18AnnH3RcmupzKCP6/fBAYlu5YQegHDgrHr+UB/M/vv\n5JYUnrt/Gvy/G3iR2HBqKigCiuL+unueWNinpFQL9zBLIUgCBTclnwDed/cHkl1PRZhZhpn9JPi6\nEbEb8ZuTW1X53P02d2/l7m2J/Y6/4e5XJ7msUMysSXDjnWBIYyCQEjPE3P1/gB1mdmawawDHLm2e\nUlLqY/ZOtBRCkssKxcz+BPQFWphZEXCXuz+R3KpC6QWMAd4Nxq4Bbg/etVzbnQo8Fcyyqgc85+4p\nNa0wBf0MeDHWJ6A+8Ky7v5LckirkRuCZoPP4ESm8lEpKTYUUEZFwUm1YRkREQlC4i4hEkMJdRCSC\nFO4iIhGkcBcRiSCFu9QoMxthZh2TXcdRZvYnM9tgZr+q5utMN7Nbgq9nmNmFZbTNNrPBcdvDUmkF\nVKkdUmqeu0TCCGAJpbw5xMzqu/vhmirEzP4F6O7u/6eSj69Uve5+ZzlNsomtBrk0aL8YvVlPKkg9\nd6kSM7s6WDO90MweC94whJl9a2a/DdZSX2dmPzOznsAw4P6g/b+Z2QozeyhY9/tmM2trZm8EvenX\nzaxNcL48M8s1swIz2xqsv4KZrTSz7Lh6VptZVoka083syWCN8bfNrF9w6FWgZVDLBSUec6LrjTez\nxWb2BvB6sG+KmeUHNd8dd447gseuBs4sce6RwdfdzWxN8DytD1YmnAFcHtR1eXDN2UH7sp6fh4Nz\nfRR3/lOD56jQzN4r+X1KdCncpdLM7CzgcqBXsFDUEeCq4HATYJ27ZwErgevdfQ2xHugUd8929w+D\ntg3dPcfd/xOYBTzl7pnAM8DDcZdsS2ydkiFArpmlE1saYXxQzxlAuru/U6LUG4itANwZGE3sXavp\nxF5oPgxqWVXKt1ja9SC23shId+9jZgOB9kG7bKCbmfU2s27Elg7IJvbh8d1Lef4aAguIrTGfBVxI\nbLGqO4EFQV0LSjysrOfnVOB8YChwX7DvSmBZ8PPJAgqROkHhLlUxAOgG5AdLEwwATg+OHSQ2/AKx\nD/hoW8Z54gPsPODZ4Os/Eguro55z9x/c/QNibw3vACwEhgaLm10D5JVy/vOB/wZw983AP4Azyv/2\nSr0exJYOPro2/8Dg39vA34M27YELgBfdfX+wimZpwypnAjvdPT+o7esQwzxlPT8vBfVuIrYMAMTW\nY5pgZtOBzsGa/FIHaMxdqsKI9SJvK+XYIf/ftS2OUPbv2nchr1dyrQx39/1m9hqxD235v8RebBLl\nuOsF/8fXa8Dv3P2x+IZm9u8JrCOs7+NLgNiHxFhsyd0hQJ6ZPeDuTyehNqlh6rlLVbwOjDSzUwDM\nrLmZ/Ws5j/kGaFrG8TXEhjMgNsQTP1wyyszqmdm/EfsLYUuwfy6x4Yl8d/+ylHOuCs51dOimTdxj\ny3Ki68VbBlxjsfXuMbOWwfOxEhhhZo0stkriz0t57BbgVDPrHjy2qZnVp+znqKzn5zjBz2OXuz9O\n7HlK2SVspWLUc5dKc/dNZvb/gFfNrB5wiNj49j/KeNh84HEzuwkYWcrxG4l9Es4UYp+KE78q3yfA\neuDHwCR3PxDU8ZaZfQ08eYJr/gF41MzeBQ4D4939e4utXFiW465X8jHu/mpw72FtcOxb4OrgM2cX\nAO8Q+xSo/JInd/eDZnY5MMtiSxL/k9i4+5vAtGCo63cVeH5K0xeYYmaHgtrGlvdNSzRoVUhJCWaW\nByxx9+dLOXYasALo4O4J+Vi6sq4nkgo0LCMpzczGEvtM1zsSFewiUaCeu4hIBKnnLiISQQp3EZEI\nUriLiESQwl1EJIIU7iIiEaRwFxGJoP8PNeOsvioPtVoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe2847edd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(\n",
    "    entropy(val_predictions[hits]), bins=30, \n",
    "    normed=True, alpha=0.7, label='correct prediction'\n",
    ");\n",
    "plt.hist(\n",
    "    entropy(val_predictions[~hits]), bins=30, \n",
    "    normed=True, alpha=0.5, label='misclassification'\n",
    ");\n",
    "plt.legend();\n",
    "plt.xlabel('entropy of predictions');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### confidence of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEKCAYAAAAyx7/DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHN1JREFUeJzt3Xt0VeW57/HvQwADyIkisaOCFuxWKcQkQFCEFrl4K1q8\nDNiWKoI3int7qW5FtK2i27rtkVE9xW4xpRi1XhBERY83UBEoKESJCIj3qKkciajxgiDE5/wxZ7ID\nJmQmWZfMld9njAzXZa45n3ct/OXNu975TnN3REQkPtqluwAREWkaBbeISMwouEVEYkbBLSISMwpu\nEZGYUXCLiMSMgltEJGYU3CIiMRMpuM3sUjNbb2brzOx+M8tOdmEiIlI/a+zMSTPrASwH+rr7N2b2\nIPCEu5c09Jru3bt7r169ElmniEhGe/nllz9x99wo27aPuM/2QCcz2wF0Bj7a08a9evWitLQ04q5F\nRMTM3o+6baNDJe7+T2AG8AGwCahy92eaX56IiLREo8FtZvsCJwO9gQOALmZ2Zj3bTTazUjMrrays\nTHylIiICRPty8hjgPXevdPcdwAJgyO4buXuxuxe5e1FubqRhGhERaYYoY9wfAIPNrDPwDTAKaPIA\n9o4dO6ioqGDbtm1NfanETHZ2Nj179qRDhw7pLkUkIzUa3O7+kpnNB14BdgJrgOKmHqiiooKuXbvS\nq1cvzKzplUosuDtbtmyhoqKC3r17p7sckYwUaR63u1/r7n3cPc/dJ7j79qYeaNu2bey3334K7Qxn\nZuy33376y0okiVJ65qRCu23Q5yySXDrlXUQkZqKegJNw55asTuj+/jZpUEL3l2glJSUcd9xxHHDA\nAUk/1qRJkzjppJMYO3Ys5513Hpdddhl9+/atd9slS5bQsWNHhgwJJgrNmjWLzp07c9ZZZyW9ThFp\nnrQFd5zs3LmT9u3bN3g/ipKSEvLy8pod3M05JsDs2bP3+PySJUvYe++9a4N7ypQpzapPJJNF7Wim\nqgPZpoZK7r77bvLz8ykoKGDChAkAlJeXM3LkSPLz8xk1ahQffPABEPRap0yZwpFHHsnUqVOZPn06\nEyZMYOjQoUyYMIHq6mquuOIKBg0aRH5+PnfccUftcf74xz9y+OGHU1BQwLRp05g/fz6lpaWcccYZ\nFBYW8s033+xS1/Dhw7nkkksoLCwkLy+PVatWAUQ+prtz4YUXcthhh3HMMcewefPmXfZds/zAU089\nxYABAygoKGDUqFGUl5cza9YsbrnlFgoLC1m2bBnTp09nxowZAJSVlTF48GDy8/M59dRT+eyzz2r3\neeWVV3LEEUdw6KGHsmzZsmR8XCLSgDbT416/fj033HADK1asoHv37nz66acAXHTRRUycOJGJEycy\nZ84cLr74Yh555BEgmMK4YsUKsrKymD59Ohs2bGD58uV06tSJ4uJicnJyWL16Ndu3b2fo0KEcd9xx\nbNy4kUcffZSXXnqJzp078+mnn9KtWzduu+02ZsyYQVFRUb31bd26lbKyMpYuXco555zDunXrACId\nc82aNbzxxhts2LCBjz/+mL59+3LOOefssv/KykrOP/98li5dSu/evWvrmjJlCnvvvTeXX345AM8+\n+2zta8466yxmzpzJ0UcfzTXXXMN1113HrbfeCgR/AaxatYonnniC6667jsWLFyf2AxORBrWZ4H7u\nuecYN24c3bt3B6Bbt24ArFy5kgULFgAwYcIEpk6dWvuacePGkZWVVXt/zJgxdOrUCYBnnnmGtWvX\nMn/+fACqqqp46623WLx4MWeffTadO3fe5TiNGT9+PADDhg3jiy++4PPPP498zKVLlzJ+/HiysrI4\n4IADGDly5Pf2/+KLLzJs2LDaudWN1VVVVcXnn3/O0UcfDcDEiRMZN25c7fOnnXYaAAMHDqS8vDxS\nG0UkMdpMcDdHly5dGrzv7sycOZPjjz9+l22efvrpZh1r9yl0NfejHPOJJ55o1jFbYq+99gIgKyuL\nnTt3pvz4Im1ZmxnjHjlyJPPmzWPLli0AtUMlQ4YM4YEHHgDg3nvv5Wc/+1mk/R1//PHcfvvt7Nix\nA4A333yTr7/+mmOPPZY777yTrVu37nKcrl278uWXXza4v7lz5wKwfPlycnJyyMnJiXzMYcOGMXfu\nXKqrq9m0aRPPP//89147ePBgli5dynvvvReprpycHPbdd9/a8et77rmntvctIumVth53qqfv9evX\nj9/+9rccffTRZGVl0b9/f0pKSpg5cyZnn302N998M7m5udx5552R9nfeeedRXl7OgAEDcHdyc3N5\n5JFHOOGEEygrK6OoqIiOHTsyevRobrzxxtovOzt16sTKlStrhz9qZGdn079/f3bs2MGcOXOadMxT\nTz2V5557jr59+3LQQQdx1FFHfe+1ubm5FBcXc9ppp/Hdd9+x//77s2jRIn7xi18wduxYHn30UWbO\nnLnLa+666y6mTJnC1q1bOfjggyO/NyKSXI1eAac5ioqKfPcLKbz++uv85Cc/SfixMsHw4cP3+MVl\nHOnzlkySiumAZvayu0cKgTYzVCIikin05WQrsGTJknSXICIxoh63iEjMKLhFRGJGwS0iEjMKbhGR\nmEnfl5PP/1di9zfiqhbvYuHChWzYsIFp06Y16XWJnM5XWlrK3XffzZ///Ge2b9/OiSeeyCeffMJV\nV13FokWL9rhEa0PKysr46KOPGD16NND8dopI69BocJvZYcDcOg8dDFzj7rcmrao0GTNmDGPGjElr\nDUVFRbW/ANasWQMEwQtw+umnN2ufZWVllJaW1gZ3a2iniDRfo0Ml7v6Guxe6eyEwENgKPJz0yhKs\nvLycPn36MGnSJA499FDOOOMMFi9ezNChQznkkENYtWoVJSUlXHjhhQDMmzePvLw8CgoKGDZsGADV\n1dVcfvnl5OXlkZ+f/70zDQEuuOACioqK6NevH9dee23t49OmTaNv377k5+fXrsRX3zGWLFnCSSed\nxObNmznzzDNZvXo1hYWFvPPOO3tcohVg1apVHHXUUfTv358hQ4bwxhtv8O2333LNNdcwd+5cCgsL\nmTt37i7t3NOythdffDFDhgzh4IMPrl3YSkTSr6lDJaOAd9z9/WQUk2xvv/028+bNY86cOQwaNIj7\n7ruP5cuXs3DhQm688UZOOeWU2m2vv/56nn76aXr06FG7Ul9xcTHl5eWUlZXRvn372vU+6vrDH/5A\nt27dqK6uZtSoUaxdu5YePXrw8MMPs3HjRsysdn/1HaPG/vvvz+zZs5kxYwaPP/74Ls/Vt0QrQJ8+\nfVi2bBnt27dn8eLFXH311Tz00ENcf/31lJaWcttttwHBRR1q7GlZ202bNrF8+XI2btzImDFjGDt2\nbAs/ARFJhKZ+OflL4P5kFJIKvXv35vDDD6ddu3b069ePUaNGYWYcfvjh31uadOjQoUyaNIm//vWv\nVFdXA7B48WJ+/etf116Jpr6lUR988EEGDBhA//79Wb9+PRs2bCAnJ4fs7GzOPfdcFixYULvka33H\niKKhJVqrqqoYN24ceXl5XHrppaxfv77Rfa1cuZJf/epXQLCs7fLly2ufO+WUU2jXrh19+/bl448/\njlyfiCRX5OA2s47AGGBeA89PNrNSMyutrKxMVH0JVbMUKUC7du1q77dr1+57S5POmjWLG264gQ8/\n/JCBAwfWriq4J++99x4zZszg2WefZe3atZx44ols27aN9u3bs2rVKsaOHcvjjz/OCSec0Oxj7Mnv\nf/97RowYwbp163jsscfYtm1bi/ZX9/1Kxpo2ItI8Telx/xx4xd3r7Xq5e7G7F7l7UW5ubmKqS6N3\n3nmHI488kuuvv57c3Fw+/PBDjj32WO64447akN99qOSLL76gS5cu5OTk8PHHH/Pkk08C8NVXX1FV\nVcXo0aO55ZZbePXVVxs8RhQNLdFaVVVFjx49gF2HQ/a0pGxzl7UVkfRpyhj3eBI5TJKA6XvJdMUV\nV/DWW2/h7owaNYqCggLy8vJ48803yc/Pp0OHDpx//vm1X/IBFBQU0L9/f/r06cOBBx7I0KFDAfjy\nyy85+eST2bZtG+7On/70pwaP8cILLzRaW0NLtE6dOpWJEydyww03cOKJJ9ZuP2LECG666SYKCwu5\n6qpd3/fmLmsrIukTaVlXM+sCfAAc7O5VjW2vZV1Fn7dkkta2rGukHre7fw3s1+yKREQkYXTKu4hI\nzKQ0uDUzoW3Q5yySXCkL7uzsbLZs2aL/qTOcu7Nlyxays7PTXYpIxkrZIlM9e/akoqKC1jrHWxIn\nOzubnj17prsMkYyVsuDu0KFD7Zl+IiLSfPpyUkQkZhTcIiIxo+AWEYkZBbeISMwouEVEYkbBLSIS\nMwpuEZGYUXCLiMSMgltEJGYU3CIiMaPgFhGJGQW3iEjMKLhFRGImUnCb2T5mNt/MNprZ62Z2VLIL\nExGR+kVd1vX/AE+5+1gz6wh0TmJNIiKyB40Gt5nlAMOASQDu/i3wbXLLEhGRhkQZKukNVAJ3mtka\nM5ttZl2SXJeIiDQgSnC3BwYAt7t7f+BrYNruG5nZZDMrNbNSXZ5MRCR5ogR3BVDh7i+F9+cTBPku\n3L3Y3YvcvSg3NzeRNYqISB2NBre7/z/gQzM7LHxoFLAhqVWJiEiDos4quQi4N5xR8i5wdvJKEhGR\nPYkU3O5eBhQluRYREYlAZ06KiMSMgltEJGYU3CIiMaPgFhGJGQW3iEjMKLhFRGJGwS0iEjMKbhGR\nmFFwi4jEjIJbRCRmFNwiIjGj4BYRiRkFt4hIzCi4RURiRsEtIhIzCm4RkZhRcIuIxIyCW0QkZiJd\nuszMyoEvgWpgp7vrMmYiImkS9WLBACPc/ZOkVSIiIpFoqEREJGaiBrcDi83sZTObXN8GZjbZzErN\nrLSysjJxFYqIyC6iBvdP3b0Q+Dnw72Y2bPcN3L3Y3YvcvSg3NzehRYqIyP+IFNzu/s/wv5uBh4Ej\nklmUiIg0rNHgNrMuZta15jZwHLAu2YWJiEj9oswq+QHwsJnVbH+fuz+V1KpERKRBjQa3u78LFKSg\nFhERiUDTAUVEYkbBLSISMwpuEZGYUXCLiMSMgltEJGYU3CIiMaPgFhGJGQW3iEjMKLhFRGJGwS0i\nEjMKbhGRmFFwi4jEjIJbRCRmFNwiIjGj4BYRiRkFt4hIzCi4RURiRsEtIhIzkYPbzLLMbI2ZPZ7M\ngkREZM+a0uO+BHg9WYWIiEg0kYLbzHoCJwKzk1uOiIg0JmqP+1ZgKvBdQxuY2WQzKzWz0srKyoQU\nJyIi39docJvZScBmd395T9u5e7G7F7l7UW5ubsIKFBGRXUXpcQ8FxphZOfAAMNLM/p7UqkREpEGN\nBre7X+XuPd29F/BL4Dl3PzPplYmISL00j1tEJGbaN2Vjd18CLElKJSIiEol63CIiMaPgFhGJGQW3\niEjMKLhFRGJGwS0iEjMKbhGRmFFwi4jEjIJbRCRmFNwiIjGj4BYRiRkFt4hIzCi4RURiRsEtIhIz\nCm4RkZhRcIuIxIyCW0QkZhTcIiIxo+AWEYmZRoPbzLLNbJWZvWpm683sulQUJiIi9YtyzcntwEh3\n/8rMOgDLzexJd38xybWJiEg9Gg1ud3fgq/Buh/DHk1mUiIg0LNIYt5llmVkZsBlY5O4v1bPNZDMr\nNbPSysrKRNcpIiKhSMHt7tXuXgj0BI4ws7x6til29yJ3L8rNzU10nSIiEmrSrBJ3/xx4HjghOeWI\niEhjoswqyTWzfcLbnYBjgY3JLkxEROoXZVbJD4G7zCyLIOgfdPfHk1uWiIg0JMqskrVA/xTUIiIi\nEejMSRGRmFFwi4jEjIJbRCRmFNwiIjGj4BYRiZko0wGluZ7/r2jbjbgquXWISEZRj1tEJGYU3CIi\nMaPgFhGJGQW3iEjMKLhFRGJGs0qaKupMERGJhXNLVqe7hCZTj1tEJGYU3CIiMaPgFhGJGQW3iEjM\nKLhFRGImyjUnDzSz581sg5mtN7NLUlGYiIjUL8p0wJ3Af7j7K2bWFXjZzBa5+4Yk1yYiIvWIcs3J\nTcCm8PaXZvY60ANQcCeKVhEUkSZo0hi3mfUiuHDwS8koRkREGhc5uM1sb+Ah4Dfu/kU9z082s1Iz\nK62srExkjSIiUkek4DazDgShfa+7L6hvG3cvdvcidy/Kzc1NZI0iIlJHo2PcZmbA34DX3f1PyS8p\nTbQGiYjERJRZJUOBCcBrZlYWPna1uz+RvLKkXvoSU0SINqtkOWApqEUSRQEvktF05qSISMwouEVE\nYkbBLSISM7oCTlumsXDJYHG8sk1U6nGLiMRM5ve4NT9bRDKMetwiIjGj4BYRiRkFt4hIzCi4RURi\nJvO/nJSW07RBkVZFwS0isZLJ87OjUnBL4jRl6qV65yLNpuCW1k3DNCLfo+CW9NCJUSLNplklIiIx\nox63iLQK+tIxOgW3tC0aM5cMEOViwXOAk4DN7p6X/JJEmkFj5tKGROlxlwC3AXcntxQRyVQaBkms\nKBcLXmpmvZJfioi0FlGD9m+TBiW5EqmPxrhFpNnUk06PhAW3mU0GJgMcdNBBidqtiEQUJUTVQ84M\nCZvH7e7F7l7k7kW5ubmJ2q2IiOxGQyUiwKNl/9zl/sL36++9qscqrUGU6YD3A8OB7mZWAVzr7n9L\ndmESH7uHXkNOLuyR5Eq+L2ptInESZVbJ+FQUIpkv0QHfVkI5kV8A6svEzKChEpGWaMGZmApRaS4F\nt+xRW+nVJlJ971lDY+YizRHf4G5jpzi35nHkRGvNvyx27yWP+SxarWM+uyvyMRbuO7FJNUnbE9/g\nlhZpzeHYGjQlaEVSTcGdYRTIIplPwS3SykTt7WtIpe1ScLcC6iWLSFPo0mUiIjGjHrdITGlIpe1S\ncDeDhjYkEyX6F4F+sSSPglskwyV6aqOmSqafxrhFRGJGPW4RiQ0NvwQU3HVo7Fok9TT00nQKbhHJ\nOJneM299wd3GFo8SEWkqfTkpIhIzra/HnSQavxaR3SV+fD011ySN1OM2sxPM7A0ze9vMpiW7KBER\naViUiwVnAX8BjgUqgNVmttDdNyS7uCjUkxaRtiZKj/sI4G13f9fdvwUeAE5OblkiItKQKMHdA/iw\nzv2K8DEREUmDhH05aWaTgcnh3e1mti5R+46h7sAn6S4ijdpy+9ty26Gtt/+yv7Sk/T+KumGU4P4n\ncGCd+z3Dx3bh7sVAMYCZlbp7UdQiMo3a33bb35bbDmp/qtofZahkNXCImfU2s47AL4GFyS1LREQa\n0miP2913mtmFwNNAFjDH3dcnvTIREalXpDFud38CeKIJ+y1uXjkZQ+1vu9py20HtT0n7zd1TcRwR\nEUkQrVUiIhIzLQruxk6Ft8Cfw+fXmtmAlhyvNYnQ9jPCNr9mZivMrCAddSZL1GUQzGyQme00s7Gp\nrC/ZorTfzIabWZmZrTezF1JdYzJF+PefY2aPmdmrYfvPTkedyWBmc8xsc0NTnlOSe+7erB+CLyrf\nAQ4GOgKvAn1322Y08CRgwGDgpeYerzX9RGz7EGDf8PbPM6XtUdtfZ7vnCL4fGZvuulP8+e8DbAAO\nCu/vn+66U9z+q4E/hrdzgU+BjumuPUHtHwYMANY18HzSc68lPe4op8KfDNztgReBfczshy04ZmvR\naNvdfYW7fxbefZFg/numiLoMwkXAQ8DmVBaXAlHa/ytggbt/AODumfQeRGm/A13NzIC9CYJ7Z2rL\nTA53X0rQnoYkPfdaEtxRToXP1NPlm9qucwl+A2eKRttvZj2AU4HbU1hXqkT5/A8F9jWzJWb2spmd\nlbLqki9K+28DfgJ8BLwGXOLu36WmvLRLeu61mfW408XMRhAE90/TXUuK3Qpc6e7fBZ2uNqc9MBAY\nBXQCVprZi+7+ZnrLSpnjgTJgJPBjYJGZLXP3L9JbVmZoSXBHORU+0unyMRSpXWaWD8wGfu7uW1JU\nWypEaX8R8EAY2t2B0Wa2090fSU2JSRWl/RXAFnf/GvjazJYCBUAmBHeU9p8N3OTBoO/bZvYe0AdY\nlZoS0yrpudeSoZIop8IvBM4Kv2UdDFS5+6YWHLO1aLTtZnYQsACYkIG9rEbb7+693b2Xu/cC5gP/\nliGhDdH+7T8K/NTM2ptZZ+BI4PUU15ksUdr/AcFfG5jZD4DDgHdTWmX6JD33mt3j9gZOhTezKeHz\nswhmE4wG3ga2EvwWjr2Ibb8G2A/477DXudMzZPGdiO3PWFHa7+6vm9lTwFrgO2C2u2fEipkRP///\nBErM7DWC2RVXuntGrBpoZvcDw4HuZlYBXAt0gNTlns6cFBGJGZ05KSISMwpuEZGYUXCLiMSMgltE\nJGYU3CIiMaPglhYxs5vD1d9uNrMp9Z3abWa9GlpJLV3M7GIze93M7k3ycYab2ePh7TGNrKS4j5n9\nW537B5jZ/GTWJ/Gk6YDSImZWBXRz9+o9bNMLeNzd81JVV2PMbCNwjLtXNOO1RvD/TqNrb5jZcOBy\ndz8pwra9aGXvk7RO6nG3UWZ2VrhW8Ktmdk/4WC8zey58/Nnw7E/MrCRcX3iFmb1bs7a2mS0kWPnt\nZTM73cymm9nl4XMDw32/Cvx7neNmhb3z1eFxfh0+PjxckGm+mW00s3vDgKxZ03tFuL9VZta1of3U\n087LzGxd+POb8LFZBEuSPmlml+62/SQzezSs5S0zu7bOe/OGmd0NrAMONLPjzGylmb1iZvPMbO9w\n2xPCNrwCnLbbvm8Lb//AzB6ueY/MbAhwE/BjC9bwvrnuXypmlm1md1qwvvsaC9bAqdnnAjN7Kqz3\nf9d5n0vCdr+2ezsl5tK9tq1+Uv8D9CNYM6N7eL9b+N/HgInh7XOAR8LbJcA8gl/0fQmW9KzZ11d1\nbk8n6F1CcMbgsPD2zYRrFwOTgd+Ft/cCSoHeBGeiVRGs69AOWEmwMFdHglOlB4Wv+V8EZ/zWu5/d\n2jmQYGW6LgS/YNYD/cPnymvav9trJgGbCM567UQQ0kVAL4IzIAeH23UHlgJdwvtXEpwtm02wMtwh\nBGcMPkjQi67Z923h7bnAb8LbWUBOeIx1dWqpvQ/8B8EZihCs+fFBeKxJ4fuTE95/n2CdjIHAojr7\n2ifd/+70k7gf9bjbppHAPA9PQXb3mrWFjwLuC2/fw64rGj7i7t+5+wbgB3vauZntQxAUS+vsq8Zx\nBOs4lAEvEQTkIeFzq9y9woMhiDKC4DoM2OTuq8Nav3D3nY3sp8ZPgYfd/Wt3/4pg7Zif7an20CJ3\n3+Lu34SvqXkf3vdgfWUIFsjvC/wjrGEi8COCUH3P3d9ydwf+3sAxRhIueevu1e5e1UhNP63Zl7tv\nJAjoQ8PnnnX3KnffRnDxhh8RhPnBZjbTzE4AtCpfBtGyrhLV9jq3W7JOqwEXufvTuzwYjAXXPUY1\ne/73We9+EmT3L35q7n+92/EXufv4XYoyK0xCPY353vvm7p9ZcLm844EpwL8S/BUlGUA97rbpOWCc\nme0HYGbdwsdXEKz0BnAGsKw5O3f3z4HPzaymp3pGnaefBi4wsw7hsQ81sy572N0bwA/NbFC4fVcz\nax9xP8uAU8ysc/jcqRHbdKyZdTOzTsApwD/q2eZFYKiZ/Ut4/C5mdiiwEehlZj8Otxtfz2sBngUu\nCF+bZWY5wJdA1wa2X0b4PobHOYjgvamXmXUH2rn7Q8DvCC61JRlCPe42yIOV3P4AvGBm1cAagrHS\ni4A7zewKoJKWrWp2NjDHzBx4ps7jswmGQF4Jv3ysJAjHhmr91sxOB2aGQfoNcEyU/bj7K2ZWwv+s\nAT3b3ddEqH0VwSXXegJ/d/dSC2Z81N13pZlNAu43s73Ch3/n7m+a2WTg/5rZVoLArS+MLwGKzexc\ngl7yBe6+0sz+EX4h+STwlzrb/zdwuwWr7e0EJrn7dmv4IhU9CD7Lms7ZVRHaLTGh6YAidYRhXOTu\nF6a7FpGGaKhERCRm1OMWEYkZ9bhFRGJGwS0iEjMKbhGRmFFwi4jEjIJbRCRmFNwiIjHz/wFJhauy\nk/5rfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe2847e978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(\n",
    "    val_predictions[hits].max(1), bins=30, \n",
    "    normed=True, alpha=0.7, label='correct prediction'\n",
    ");\n",
    "plt.hist(\n",
    "    val_predictions[~hits].max(1), bins=30, \n",
    "    normed=True, alpha=0.5, label='misclassification'\n",
    ");\n",
    "plt.legend();\n",
    "plt.xlabel('confidence of predictions');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### difference between biggest and second biggest probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEKCAYAAAAyx7/DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGRxJREFUeJzt3Xt0VeW57/HvQwINCDsViXtUkA3uoVIIuWBQJMrVW8Hi\nZcBxW4uAF0rP8XLaowjtqEU2te2QUR3FHpFSRN26pSAqZagIKoVUNASNFBC81GxNywFExQuiEJ/z\nx1pkc8llJqy5Vt7k9xkjw6ysud75vKz4W2/eOec7zd0REZFwtMt0ASIi0jQKbhGRwCi4RUQCo+AW\nEQmMgltEJDAKbhGRwCi4RUQCo+AWEQmMgltEJDDZcTTarVs379WrVxxNi4i0Shs2bPjA3fOibBtL\ncPfq1YuKioo4mhYRaZXM7L+ibqupEhGRwCi4RUQCo+AWEQlMLHPcddm/fz/V1dXs27cvXbuUDMnJ\nyaFHjx60b98+06WItEppC+7q6mq6dOlCr169MLN07VbSzN3ZvXs31dXV9O7dO9PliLRKaZsq2bdv\nHyeccIJCu5UzM0444QT9ZSUSo7TOcSu02wa9zyLx0sFJEZHApG2O+0jXLlyf0vb+MHFgSttLtYUL\nF3LBBRdw0kknxb6viRMncvHFFzN27Fiuu+46fvzjH9O3b986t129ejUdOnRg8ODBAMydO5dOnTpx\n9dVXx16niDRPxoI7JAcOHCA7O7vex1EsXLiQ/Pz8Zgd3c/YJMH/+/AafX716NZ07d64N7ilTpjSr\nPpHWLOpAM10DyDY1VfLQQw9RUFBAYWEh48ePB6CqqooRI0ZQUFDAyJEjee+994DEqHXKlCmcddZZ\nTJ06lRkzZjB+/HhKS0sZP348NTU13HrrrQwcOJCCggLuv//+2v38+te/pn///hQWFjJt2jSWLFlC\nRUUFV111FUVFRXzxxReH1TVs2DBuvvlmioqKyM/Pp7y8HCDyPt2dG264gdNPP53zzjuPnTt3Htb2\nweUHnn32WQYMGEBhYSEjR46kqqqKuXPncvfdd1NUVMTatWuZMWMGs2fPBqCyspJBgwZRUFDAZZdd\nxkcffVTb5m233caZZ57Jaaedxtq1a+N4u0SkHm1mxL1582ZmzZrFSy+9RLdu3fjwww8BuPHGG5kw\nYQITJkxgwYIF3HTTTTz55JNA4hTGl156iaysLGbMmMGWLVsoKyujY8eOzJs3j9zcXNavX8+XX35J\naWkpF1xwAVu3buWpp57ilVdeoVOnTnz44Yd07dqVe++9l9mzZ1NSUlJnfXv37qWyspI1a9ZwzTXX\nsGnTJoBI+3zttdfYtm0bW7ZsYceOHfTt25drrrnmsPZ37drF9ddfz5o1a+jdu3dtXVOmTKFz587c\ncsstADz//PO1r7n66quZM2cOQ4cO5fbbb+eOO+7gnnvuARJ/AZSXl/P0009zxx13sGrVqtS+YSJS\nrzYT3C+88ALjxo2jW7duAHTt2hWAdevWsXTpUgDGjx/P1KlTa18zbtw4srKyah+PGTOGjh07AvDc\nc8+xceNGlixZAsCePXt46623WLVqFZMmTaJTp06H7acxV155JQBDhgzhk08+4eOPP468zzVr1nDl\nlVeSlZXFSSedxIgRI45q/+WXX2bIkCG151Y3VteePXv4+OOPGTp0KAATJkxg3Lhxtc9ffvnlAJxx\nxhlUVVVF6qOIpEabCe7mOO644+p97O7MmTOHCy+88LBtVqxY0ax9HXkK3cHHUfb59NNPN2ufx+Ib\n3/gGAFlZWRw4cCDt+xdpy9rMHPeIESNYvHgxu3fvBqidKhk8eDCPPfYYAI888gjnnntupPYuvPBC\n7rvvPvbv3w/Am2++yeeff87555/PAw88wN69ew/bT5cuXfj000/rbW/RokUAlJWVkZubS25ubuR9\nDhkyhEWLFlFTU8P27dt58cUXj3rtoEGDWLNmDe+++26kunJzczn++ONr568ffvjh2tG3iGRWpBG3\nmX0TmA/kAw5c4+7rjmXH6T59r1+/fvz0pz9l6NChZGVlUVxczMKFC5kzZw6TJk3irrvuIi8vjwce\neCBSe9dddx1VVVUMGDAAdycvL48nn3ySiy66iMrKSkpKSujQoQOjRo3izjvvrD3Y2bFjR9atW1c7\n/XFQTk4OxcXF7N+/nwULFjRpn5dddhkvvPACffv2pWfPnpx99tlHvTYvL4958+Zx+eWX8/XXX3Pi\niSeycuVKvvvd7zJ27Fieeuop5syZc9hrHnzwQaZMmcLevXs55ZRTIv/biEi8zN0b38jsQWCtu883\nsw5AJ3f/uL7tS0pK/MgbKbzxxht8+9vfPtZ6W6Vhw4Y1eOAyRHq/pTVJx+mAZrbB3SOFQKMjbjPL\nBYYAEwHc/Svgq2ZXJyIixyTKVElvYBfwgJkVAhuAm93981gra0NWr16d6RJEJCBRDk5mAwOA+9y9\nGPgcmHbkRmY22cwqzKxi165dKS5TREQOihLc1UC1u7+SfLyERJAfxt3nuXuJu5fk5UW6UbGIiDRD\no8Ht7v8PeN/MTk/+aCSwJdaqRESkXlEvwLkReCR5RsnfgEnxlSQiIg2JFNzuXgmk9ly1F3+Z0uYY\nPv2Ym1i2bBlbtmxh2rSjpvAblMrT+SoqKnjooYf47W9/y5dffsno0aP54IMPmD59OitXrmxwidb6\nVFZW8o9//INRo0YBze+niLQMuuT9EGPGjGHMmDEZraGkpKT2A+C1114DEsELcMUVVzSrzcrKSioq\nKmqDuyX0U0Sar81c8l5VVUWfPn2YOHEip512GldddRWrVq2itLSUU089lfLychYuXMgNN9wAwOLF\ni8nPz6ewsJAhQ4YAUFNTwy233EJ+fj4FBQVHXWkI8MMf/pCSkhL69evHz3/+89qfT5s2jb59+1JQ\nUFC7El9d+1i9ejUXX3wxO3fu5Pvf/z7r16+nqKiId955p8ElWgHKy8s5++yzKS4uZvDgwWzbto2v\nvvqK22+/nUWLFlFUVMSiRYsO62dDy9redNNNDB48mFNOOaV2YSsRybw2NeJ+++23Wbx4MQsWLGDg\nwIE8+uijlJWVsWzZMu68804uvfTS2m1nzpzJihUr6N69e+1KffPmzaOqqorKykqys7Nr1/s41C9+\n8Qu6du1KTU0NI0eOZOPGjXTv3p0nnniCrVu3Yma17dW1j4NOPPFE5s+fz+zZs1m+fPlhz9W1RCtA\nnz59WLt2LdnZ2axatYqf/OQnPP7448ycOZOKigruvfdeIHFTh4MaWtZ2+/btlJWVsXXrVsaMGcPY\nsWOP8R0QkVRoMyNugN69e9O/f3/atWtHv379GDlyJGZG//79j1qatLS0lIkTJ/L73/+empoaAFat\nWsUPfvCD2jvR1LU06h//+EcGDBhAcXExmzdvZsuWLeTm5pKTk8O1117L0qVLa5d8rWsfUdS3ROue\nPXsYN24c+fn5/OhHP2Lz5s2NtrVu3Tq+973vAYllbcvKymqfu/TSS2nXrh19+/Zlx44dkesTkXi1\nqeA+uBQpQLt27Woft2vX7qilSefOncusWbN4//33OeOMM2pXFWzIu+++y+zZs3n++efZuHEjo0eP\nZt++fWRnZ1NeXs7YsWNZvnw5F110UbP30ZCf/exnDB8+nE2bNvGnP/2Jffv2HVN7h/57RVnTRkTS\no00Fd1O88847nHXWWcycOZO8vDzef/99zj//fO6///7akD9yquSTTz7huOOOIzc3lx07dvDMM88A\n8Nlnn7Fnzx5GjRrF3Xffzeuvv17vPqKob4nWPXv20L17d+Dw6ZCGlpRt7rK2IpI5mZvjTsHpe3G6\n9dZbeeutt3B3Ro4cSWFhIfn5+bz55psUFBTQvn17rr/++tqDfACFhYUUFxfTp08fTj75ZEpLSwH4\n9NNPueSSS9i3bx/uzm9+85t69/HnP/+50drqW6J16tSpTJgwgVmzZjF69Oja7YcPH86vfvUrioqK\nmD798H/35i5rKyKZE2lZ16bSsq6i91tak5a2rKumSkREAqPgFhEJTFqDW2cmtA16n0XilbbgzsnJ\nYffu3fqfupVzd3bv3k1OTk6mSxFptdJ2VkmPHj2orq5GN1lo/XJycujRo0emyxBptdIW3O3bt6+9\n0q9BUVcNbOGnE4qIxEUHJ0VEAqPgFhEJjIJbRCQwCm4RkcAouEVEAqPgFhEJjIJbRCQwCm4RkcAo\nuEVEAqPgFhEJTKRL3s2sCvgUqAEORF3sW0REUq8pa5UMd/cPYqtEREQi0VSJiEhgoga3A6vMbIOZ\nTa5rAzObbGYVZlahpVtFROITNbjPcfci4DvA/zKzIUdu4O7z3L3E3Uvy8vJSWqSIiPy3SMHt7n9P\n/ncn8ARwZpxFiYhI/RoNbjM7zsy6HPweuADYFHdhIiJStyhnlfwz8ISZHdz+UXd/NtaqRESkXo0G\nt7v/DShMQy0iIhKBTgcUEQmMgltEJDAKbhGRwCi4RUQCo+AWEQmMgltEJDAKbhGRwCi4RUQCo+AW\nEQmMgltEJDAKbhGRwCi4RUQCo+AWEQmMgltEJDAKbhGRwCi4RUQCo+AWEQmMgltEJDAKbhGRwCi4\nRUQCo+AWEQmMgltEJDAKbhGRwCi4RUQCEzm4zSzLzF4zs+VxFiQiIg1ryoj7ZuCNuAoREZFoIgW3\nmfUARgPz4y1HREQaE3XEfQ8wFfi6vg3MbLKZVZhZxa5du1JSnIiIHK3R4Dazi4Gd7r6hoe3cfZ67\nl7h7SV5eXsoKFBGRw0UZcZcCY8ysCngMGGFm/xFrVSIiUq9Gg9vdp7t7D3fvBfwb8IK7fz/2ykRE\npE46j1tEJDDZTdnY3VcDq2OpREREItGIW0QkMApuEZHAKLhFRAKj4BYRCUyTDk62KC/+Mtp2w6fH\nW4eISJppxC0iEhgFt4hIYBTcIiKBUXCLiARGwS0iEhgFt4hIYBTcIiKBUXCLiARGwS0iEhgFt4hI\nYBTcIiKBUXCLiARGwS0iEhgFt4hIYBTcIiKBUXCLiARGwS0iEhgFt4hIYBoNbjPLMbNyM3vdzDab\n2R3pKExEROoW5Z6TXwIj3P0zM2sPlJnZM+7+csy1iYhIHRoNbnd34LPkw/bJL4+zKBERqV+kOW4z\nyzKzSmAnsNLdX4m3LBERqU+UqRLcvQYoMrNvAk+YWb67bzp0GzObDEwG6NmzZ8oLbbYXfxltu+HT\n461DRCRFmnRWibt/DLwIXFTHc/PcvcTdS/Ly8lJVn4iIHCHKWSV5yZE2ZtYROB/YGndhIiJStyhT\nJd8CHjSzLBJB/0d3Xx5vWSIiUp8oZ5VsBIrTUIuIiESgKydFRAKj4BYRCYyCW0QkMApuEZHAKLhF\nRAKj4BYRCYyCW0QkMApuEZHAKLhFRAKj4BYRCYyCW0QkMApuEZHAKLhFRAKj4BYRCUykW5e1CbrF\nmYgEQiNuEZHAKLhFRAKj4BYRCYyCW0QkMApuEZHAKLhFRAKj4BYRCYyCW0QkMApuEZHANBrcZnay\nmb1oZlvMbLOZ3ZyOwkREpG5RLnk/APwfd3/VzLoAG8xspbtvibk2ERGpQ6Mjbnff7u6vJr//FHgD\n6B53YSIiUrcmzXGbWS+gGHgljmJERKRxkYPbzDoDjwP/290/qeP5yWZWYWYVu3btSmWNIiJyiEjB\nbWbtSYT2I+6+tK5t3H2eu5e4e0leXl4qaxQRkUNEOavEgD8Ab7j7b+IvSUREGhJlxF0KjAdGmFll\n8mtUzHWJiEg9Gj0d0N3LAEtDLSIiaXXtwvWZLqFZdOWkiEhgFNwiIoHRzYKbKupNhUE3FhaRWGjE\nLSISGAW3iEhgFNwiIoFRcIuIBEbBLSISGAW3iEhgFNwiIoFRcIuIBEbBLSISGAW3iEhgdMl7nKJe\nHq9L40VSKtRV/6LSiFtEJDAacbcEGpmLSBNoxC0iEhgFt4hIYBTcIiKBUXCLiARGwS0iEhgFt4hI\nYBTcIiKBUXCLiASm0eA2swVmttPMNqWjIBERaViUKycXAvcCD8VbiohIw1r7GiRRNRrc7r7GzHrF\nX4o0SpfGiwgtcK2Spyr/Hmm7S4q6x1xJwBTwIq1ayoLbzCYDkwF69uyZqmbrpYAXaR00/dF0KQtu\nd58HzAMoKSnxVLV7rBTwItLatLipEhFp2TRCzrwopwP+J7AOON3Mqs3s2vjLEhGR+kQ5q+TKdBSS\naW1ySiXqQcyodLBTJC105aSISGAU3CIigdHBSZFWLurBxD9MHBhzJZIqCm5JHV34I5IWCu6A6ABq\nA/RhIG2IgrsFiBrIqW4v1QEfdb/L/mt9yv8s13TAsdP52eFQcDdRU0K2VY18Uyxy0P5LzIXUI9Uf\nBHGEoj6E2i4Ft7RokT8oj4+2WWsaVbamvkjTKLhjlOopkFTL1JTKmI8eTGl7Im2Nglsa1dI/gDJJ\no17JBAW3tApRR/HLjp8QcyUi8dOVkyIigVFwi4gERlMlInXQ1Iu0ZApukWOQyYDXh0vbpeCWNkWn\nIkproOAWaUH0wSJRKLhF0kCBLKmk4BYRQHPmIVFwi0iTpDrg9YHRdApukVautUzTNKUfrT3kFdwi\nEovW8oHREim4RaTVae3TLwpuEWmzQg34SGuVmNlFZrbNzN42s2lxFyUiIvVrdMRtZlnA74DzgWpg\nvZktc/ctcRcnItISRJ+vT8/t5KKMuM8E3nb3v7n7V8BjwCXxliUiIvWJEtzdgfcPeVyd/JmIiGRA\nyg5OmtlkYHLy4Wdmtq2ZTXUDPkhNVcFQn1u/ttZfaIt9/vHvjqXP/xJ1wyjB/Xfg5EMe90j+7DDu\nPg+YF3XH9TGzCncvOdZ2QqI+t35trb+gPscpylTJeuBUM+ttZh2AfwOWxVuWiIjUp9ERt7sfMLMb\ngBVAFrDA3TfHXpmIiNQp0hy3uz8NPB1zLQcd83RLgNTn1q+t9RfU59iYu6djPyIikiK6y7uISGAy\nEtyNXUJvCb9NPr/RzAZkos5UitDnq5J9/auZvWRmhZmoM5WiLpVgZgPN7ICZjU1nfXGI0mczG2Zm\nlWa22cz+nO4aUy3C73aumf3JzF5P9nlSJupMFTNbYGY7zWxTPc/Hn1/untYvEgc43wFOAToArwN9\nj9hmFPAMYMAg4JV015mBPg8Gjk9+/5220OdDtnuBxDGUsZmuOw3v8zeBLUDP5OMTM113Gvr8E+DX\nye/zgA+BDpmu/Rj6PAQYAGyq5/nY8ysTI+4ol9BfAjzkCS8D3zSzb6W70BRqtM/u/pK7f5R8+DKJ\n8+VDFnWphBuBx4Gd6SwuJlH6/D1gqbu/B+Duofc7Sp8d6GJmBnQmEdwH0ltm6rj7GhJ9qE/s+ZWJ\n4I5yCX1ru8y+qf25lsQndsga7bOZdQcuA+5LY11xivI+nwYcb2arzWyDmV2dturiEaXP9wLfBv4B\n/BW42d2/Tk95GRF7fmk97hbGzIaTCO5zMl1LGtwD3ObuXycGY21CNnAGMBLoCKwzs5fd/c3MlhWr\nC4FKYATwr8BKM1vr7p9ktqxwZSK4o1xCH+ky+4BE6o+ZFQDzge+4++401RaXKH0uAR5LhnY3YJSZ\nHXD3J9NTYspF6XM1sNvdPwc+N7M1QCEQanBH6fMk4FeemAB+28zeBfoA5ekpMe1iz69MTJVEuYR+\nGXB18ujsIGCPu29Pd6Ep1GifzawnsBQY30pGX4322d17u3svd+8FLAH+Z8ChDdF+t58CzjGzbDPr\nBJwFvJHmOlMpSp/fI/EXBmb2z8DpwN/SWmV6xZ5faR9xez2X0JvZlOTzc0mcYTAKeBvYS+ITO1gR\n+3w7cALwf5Mj0AMe8AI9EfvcqkTps7u/YWbPAhuBr4H57l7naWUhiPg+/zuw0Mz+SuJMi9vcPdhV\nA83sP4FhQDczqwZ+DrSH9OWXrpwUEQmMrpwUEQmMgltEJDAKbhGRwCi4RUQCo+AWEQmMgltaLDOb\nYWa3mNlMMzsv+bNzkyvMVZpZRzO7K/n4rkzXK5IuuuRdWjx3v/2Qh1cBv3T3/wAws8lAV3evidKW\nmWW7e7ALHImARtzSwpjZT83sTTMrI3GFHWa20MzGmtl1wP8A/t3MHjGzZSRWm9tgZleYWZ6ZPW5m\n65NfpcnXzzCzh83sL8DDZpaVHKmvT66X/IPkdsOSiz8tMbOtyX1Y8rmBllgn/XUzKzezLvW1IxI3\njbilxTCzM0hcMl1E4nfzVWDDwefdfb6ZnQMsd/clydd85u5Fye8fBe5297LkEgIrSKxKB9AXOMfd\nv0iO0ve4+0Az+wbwFzN7LrldMdCPxEp2fwFKzawcWARc4e7rzeyfgC9ILAZ2VDvu/m5c/0YioOCW\nluVc4Al33wuQHFE3xXlA30NWGvwnM+uc/H6Zu3+R/P4CoMD++447ucCpwFdAubtXJ/dfCfQC9gDb\n3X09wMFV7cysvnYU3BIrBbe0Ju2AQe6+79AfJoP880N/BNzo7iuO2G4Y8OUhP6qh4f9H6mxHJG6a\n45aWZA1wafJskS7Ad5v4+udI3FEHADMrqme7FcAPzax9crvTzOy4BtrdBnzLzAYmt+9iZtnNaEck\nJTTilhbD3V81s0Uk7lu4k8SSoU1xE/A7M9tI4nd7DTClju3mk5gCeTV58HEXcGkDdX1lZlcAc8ys\nI4n57fOa2o5Iqmh1QBGRwGiqREQkMApuEZHAKLhFRAKj4BYRCYyCW0QkMApuEZHAKLhFRAKj4BYR\nCcz/B+LdKFV53Jj/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe250c2f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sorted_correct = np.sort(val_predictions[hits], 1)\n",
    "sorted_incorrect = np.sort(val_predictions[~hits], 1)\n",
    "\n",
    "plt.hist(\n",
    "    sorted_correct[:, -1] - sorted_correct[:, -2], bins=30, \n",
    "    normed=True, alpha=0.7, label='correct prediction'\n",
    ");\n",
    "plt.hist(\n",
    "    sorted_incorrect[:, -1] - sorted_incorrect[:, -2], bins=30, \n",
    "    normed=True, alpha=0.5, label='misclassification'\n",
    ");\n",
    "plt.legend();\n",
    "plt.xlabel('difference');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### probabilistic calibration of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XdclfX7x/HXBYiKIO4t4DZDTcXR0Ay3DSszK3JWDtpZ\nafmrLNO2ld9Cw3JUmOVIzcyBmpojZ+69UBy4WbI/vz/OUVEBD8jhHOB6Ph484px7XdzJeXOv6yPG\nGJRSSqnMuDi6AKWUUs5Ng0IppVSWNCiUUkplSYNCKaVUljQolFJKZUmDQimlVJY0KFSBICIjRORn\n6/c+IhIrIq42LNdWRI5lMX28iLyT0bwiskNE2uZC+Uo5NTdHF6BUbjPGRACeubSuQVlMu/3y9yIy\nAqhtjHk6N7arlDPRIwrl9ERE/6CxkS1HUUpllwaFckoiclhEhorIViBORNxEpIqIzBSR0yJySERe\nymRZPxExlwNGRPqJyC4RiRGRgyIyMINl3haRM9btBqV7f7KIfJhFje1FpDPwNtDTespri4j0EJGN\n183/mojMyWRdZURkkogcF5HzIjLb+n5fEfnnunmNiNROV984EZkvInHA6yJyMn1giMgj1v2IiLiI\nyDAROSAiZ0XkNxEpk1FNSl2mQaGc2ZPA/UApIA34A9gCVAXaAa+ISCcb1hMFPACUBPoBX4pI03TT\nKwHlrOvtA4SKSD1bizTGLABGA78aYzyNMY2BuUANEbkt3ay9gB8zWc1PgAdwO1AB+NLW7QNPAaMA\nL+BrIA4IvG76VOv3LwIPA/cCVYDzwLfZ2JYqhDQolDMba4w5aoy5BDQHyhtjPjDGJBljDgITgCdu\nthJjzJ/GmAPGYjmwCGh93WzvGGMSrdP/BB6/lcKNMYnAr8DTACJyO+AHzLt+XhGpDHQBBhljzhtj\nkq112GqOMWaVMSbNGJMA/IIlZBERL6Cr9T2AQcBwY8wxa40jgMf09J7KigaFcmZH033vC1QRkQuX\nv7Cc7ql4s5WISBcRWSsi56zLdcVyBHHZeWNMXLrXR7D8tX2rpgBPiYhgOZr4zfrhfL3qwDljzPkc\nbufoda+nAo+KSFHgUWCTMeaIdZov8Hu6fbgLSMWG/agKL/0rQjmz9K2NjwKHjDF1srMC64flTKA3\nlr+8k63n/yXdbKVFpES6sPABtt9CrZY3jFkrIklYjl6esn5l5ChQRkRKGWMuXDctDsspqcs/T6Wb\nbdsYs1NEjmA5Skl/2unytvobY1bd5OdR6go9olD5xTogxnqBu7iIuIqIv4g0v8ly7kBR4DSQIiJd\ngI4ZzPe+iLiLSGss1zOmZ7O+U4CfiFz/O/Uj8A2QbIz558bFwBhzAvgLCBGR0iJSRETaWCdvAW4X\nkTtEpBiWU0W2mAq8DLTh2p9lPDBKRHwBRKS8iHSzcZ2qkNKgUPmCMSYVywf4HcAh4AzwPeB9k+Vi\ngJeA37BcuH0Ky4Xm9E5apx0HwrBcK9idzRIvfxifFZFN6d7/CfAHfr7J8r2AZGA3lovvr1jr3wt8\nAIQD+4AMwyYDv2C5YL3UGHMm3ftfY/n5F4lIDLAWaGnjOlUhJTpwkVL2IyLFsXzwNzXG7HN0PUrl\nhB5RKGVfg4H1GhIqP7NbUIjIRBGJEpEMLwqKxVgR2S8iW6+7r12pfE9EDmO5TjDEwaUodUvseUQx\nGeicxfQuQB3r1wBgnB1rUSrPGWP8jDG+xpjNjq5FqVtht6AwxqwAzmUxSzfgR+tDUGuBUtYHj5RS\nSjkRRz5HUZVrHxQ6Zn3vxPUzisgALEcdlChRoln9+vXzpECllMrvomISOR2TSMKJfWeMMeVzso58\n8cCdMSYUCAUICAgwGzZscHBFSinl3IwxiAiLd55i5b7TjHy44ZGbL5UxR971FImldcFl1azvKaWU\nyqGL8cm8OWML3y7bD0CHBhX5oJv/La3TkUExF+htvfupFXDR+oSqUkqpHFiw/STtv1zOzE2RJKfm\n3jNydjv1JCK/AG2BcmIZPvI9oAiAMWY8MB9Lc7b9QDyW9s9KKaWy6XRMIiPm7uDPbSdoULkkk/o2\nx79qlk0LssVuQWGMefIm0w3wvL22r5RShcWJi5dYujuKNzrVY0CbmhRxzd2TRfniYrZSSqlrHTsf\nz5JdUfS5y49G1UqxelggpUu422VbGhRKKZWPpKUZfv73CJ/8Zelb2cW/EhVKFrNbSIAGhVJK5RsH\nTscybOZW1h8+T5u65Rn9iD8VShaz+3a1KaBSSuUDl5JS6TF+DXtPxfJ5j8Z0dN/HPU0a4OLigp+f\nH2FhYXbbth5RKKWUEzt4OpYa5UpQ3N2VMY83pkGVkiyeO5OBAwcQHx8PwJEjRxgwYAAAQUFBuV6D\nHlEopZQTSkhO5bOFu+nw5Qpm/2d5FrltvQpU8CrG8OHDr4TEZfHx8QwfPtwutegRhVJKOZkNh8/x\n5sytHDwdR49m1QisV/Ga6RERERkul9n7t0qDQimlnMjYJfv4MnwvVbyL82P/FrSpe20fv7S0NEqU\nKEFsbOwNy/r4+NilJg0KpZRyApeb+DWoXJI+d/rxRqd6lCh67Ud0amoqzz33HLGxsbi5uZGSknJl\nmoeHB6NGjbJLbXqNQimlHOhCfBKv/fYf/1tqaeLXvkFFRjx0+w0hkZKSQp8+fZg0aRLvvfcekydP\nxtfXFxHB19eX0NBQu1zIBj2iUEoph5m/7QTvztnOhfhkXgysk+l8ycnJBAUFMX36dEaNGsXbb78N\n2OcOp4xoUCilVB6Lik7g3Tk7WLDjJA2revNj/5Y0qFIyw3kTExN54oknmD17Np9//jlDhuT9EOwa\nFEoplcdORSeyYt9phnWpz7P31MAtkyZ+CQkJdO/enfnz5zN27FhefPHFPK7UQoNCKaXywNFz8SzZ\ndYq+d9egYTVv1gxrh7dHkUznj4+P5+GHH2bx4sV89913Vx6ocwQNCqWUsqPUNMOPaw7z2cI9uIjQ\ntVFlKngVyzIkYmNjefDBB1m+fDkTJ06kXz/HDtejQaGUUnayPyqGoTO3sfHIee6tW57RjzakglfW\nTfyio6Pp2rUra9as4aeffsqzC9ZZ0aBQSik7uJSUyuPfrSXNGMY83phHmlRFRLJc5sKFC3Tu3JmN\nGzcybdo0evTokUfVZk2DQimlctH+qFhqlbc08fuq5x3cVrkk5b2K3nS5s2fP0rFjR7Zt28aMGTPo\n1q1bHlRrG33gTimlckFCciof/bWLjl8uv9LEr03d8jaFRFRUFIGBgezYsYPZs2c7VUiAHlEopdQt\n+/fgWYbN2sahM3E80bw6gfUr3nwhqxMnTtC+fXsOHTrEH3/8QYcOHexYac5oUCil1C34KnwvX4Xv\no3qZ4oQ925K7a5ezednIyEgCAwOJjIxk/vz5tG3b1n6F3gINCqWUyoHLTfwaVfPmmXtqMKRjXTzc\nbf9IPXLkCIGBgZw+fZqFCxdy991327HaW6NBoZRS2XAuLomR83biV7YEL7evQ2D9itk61QRw8OBB\n7rvvPi5evMjixYtp2bKlnarNHXoxWymlbGCMYd7W43QYs5w/thzHJes7XTO1d+9e2rRpQ2xsLEuX\nLnX6kAA9olBKqZs6FZ3A/83ezuKdp2hUzZufn23JbZUzbuKXlZ07d9KuXTtSU1NZtmwZjRo1skO1\nuU+DQimlbuJ0TCJrDpzl7a716X935k38srJ161bat2+Pq6srf//9Nw0aNLBDpfahQaGUUhmIOBvP\n4l2neOaeGvhX9WbVsEC8i2fenykrmzZtokOHDhQvXpylS5dSt27dXK7WvjQolFIqndQ0w6RVh/h8\n0R6KuLjwYGNrE78chsS6devo1KkTJUuWZOnSpdSqVSuXK7Y/DQqllLLaeyqGN2ds5b+jFwisX4FR\nj/jftIlfVlavXk3nzp0pX748S5cuxdfXNxerzTsaFEophaWJX8/v1iAifP3EHTzUuMpNm/hlZfny\n5dx///1UrVqVJUuWUK1atVysNm9pUCilCrV9p2KoXcGT4u6u/O/JptxW2Yuynjfvz5SV8PBwHnro\nIWrUqEF4eDiVK1fOpWodQ5+jUEoVSpeSUhk9fxedvlrB75stTfzuqVPulkPir7/+4oEHHqB27dos\nW7Ys34cE6BGFUqoQWnPgLG/N2srhs/E81dKH9g2y92R1ZubOnUuPHj3w9/dn0aJFlC1bNlfW62ga\nFEqpQmXM4r2MXbIP37IeTH2uJXfVsr2JX1ZmzJjBk08+SdOmTVm4cCGlSpXKlfU6Aw0KpVShcLmJ\n3x3VvXmudQ1e61CP4u6uubLuqVOn0rt3b1q1asX8+fMpWTL7T207M7teoxCRziKyR0T2i8iwDKZ7\ni8gfIrJFRHaIiGNHEFdKFThnYxN56ZfNfL1kHwCB9Ssy/P4GuRYSU6ZM4emnn6Z169YsWLCgwIUE\n2DEoRMQV+BboAjQAnhSR659Zfx7YaYxpDLQFvhARd3vVpJQqPIwxzPkvkvZjlvPX9hMUyUHbjZuZ\nMGEC/fr1o3379vz55594enrm+jacgT1PPbUA9htjDgKIyDSgG7Az3TwG8BLLzcqewDkgxY41KaUK\ngRMXL/F/v29nye4o7qheik8fa0Tdil65uo1vv/2WF154ga5duzJz5kyKFcv5g3nOzp5BURU4mu71\nMeD6frrfAHOB44AX0NMYk3b9ikRkADAAwMfHxy7FKqUKjrOxSaw7dI7/u/82+t1dA9ec9gTPxJgx\nYxgyZAjdunXj119/pWjRW7ul1tk5+jmKTsB/QBXgDuAbEbnhBJ8xJtQYE2CMCShfvnxe16iUygcO\nn4nj+5UHAfCv6s3qtwJ5tnXNXA+Jjz76iCFDhtCjRw+mT59e4EMC7BsUkUD1dK+rWd9Lrx8wy1js\nBw4B9e1Yk1KqgElJTSN0xQE6fbWCr5fs43RMIgBexXLWxC8zxhjef/993n77bYKCgpg6dSpFiuTu\nNpyVPU89rQfqiEgNLAHxBPDUdfNEAO2AlSJSEagHHLRjTUqpAmT3yWiGztjKlmMXaX9bRT582J/y\nXrn3F35YWBjDhw8nIiICLy8voqOj6du3L99//z2urrlz11R+YLegMMakiMgLwELAFZhojNkhIoOs\n08cDI4HJIrINEGCoMeaMvWpSShUcl5JSeTJ0LS4i/O/JJjzQqPItNfG7XlhYGAMGDCA+Ph6A6Oho\n3NzcaNeuXaEKCQAxxji6hmwJCAgwGzZscHQZSikH2XMyhroVPRERVu0/w22VS1KmRO7fVe/n58eR\nI0dueN/X15fDhw/n+vbsTUQ2GmMCcrKsoy9mK6WUTeKTUhg5byedv77axO/u2uVyPSSMMaxduzbD\nkACIiIjI1e3lB9rCQynl9FbtP8OwWVs5eu4SvVr50iGXmvilFxcXx9SpUwkJCeG///5DRMjojEth\nvEVfg0Ip5dS+WLSH/y3dT41yJfh1QCta1szdjqy7du1i3LhxTJkyhejoaBo1asS4ceMoUqQIL730\n0pVrFAAeHh6MGjUqV7efH2hQKKWcUlqawcVFaOpbmoH31uTV9nUpViR3LiInJSUxe/Zsxo0bx99/\n/427uzs9evRg8ODB3HXXXVcuihcrVuzKXU8+Pj6MGjWKoKCgXKkhP9GL2Uopp3ImNpERc3dQs7wn\nr3Wom6vrPnr0KKGhoXz//fecPHkSPz8/Bg0aRP/+/SnoD/PeysVsPaJQSjkFYwyz/4vk/T92Ep+Y\nyqsdvHNlvWlpaYSHhxMSEsIff/yBMYb777+fwYMH06lTp0J3q2tOaFAopRzu+IVLDP99G8v2nKap\nTyk+6d6IOrfYxO/s2bNMnjyZ8ePHs3//fsqXL8+bb77JwIED8fPzy53CCwkNCqWUw52PT2LDkfO8\n92ADet/pl+P+TMYY1q9fT0hICL/++isJCQncc889vP/++3Tv3r1Q9GWyBw0KpZRDHDwdS/iuUwxo\nU4vbq3iz5q12eBbN2UdSfHw8v/zyCyEhIWzatAlPT0/69evH4MGDadiwYS5XXvhoUCil8lRKahoT\nVh7iy/C9FHNz4ZEm1SjvVTRHIbF7927Gjx/P5MmTuXjxIv7+/oSEhPD000/j5ZW7408UZhoUSqk8\ns/N4NG/O3ML2yGg63V6Rkd2y38QvOTmZOXPmMG7cOJYuXUqRIkV47LHHCA4O5u67787Vfk/KQlt4\nKKXyxKWkVIK+X8vJi4mMC2rKd70CqFAy41HhwsLC8PPzw8XFBT8/P8LCwoiMjGTEiBH4+vrSo0cP\nDhw4wOjRozl69ChTp07lnnvu0ZCwE32OQillV7tORFO/khciwuoDZ2hQuSSlPDLvz3R911YAV1dX\n0tIsg1927tyZ4OBgunTpore2ZoM+R6GUcjpxiSl8tnAPU9Yc5vPHGtO9WTXuqlXupssNHz78mpAA\nSE1NpWTJkmzevJmaNWvaqWKVGQ0KpVSuW7nvNG/N2sax85foc6cvnfwr2bxsZt1ZY2JiNCQcRINC\nKZWrPlu4m2+XHaBm+RJMH3Qnzf3K2LxsXFwcRYsWJSEh4YZphbFrq7PQoFBK5YrLTfwC/MoQ3BZe\nalcnW038Lly4wP33309CQgLu7u4kJSVdmVZYu7Y6C73rSSl1S6JiEhj880a+Ct8LwH31KvBm5/rZ\nConTp08TGBjI+vXrmT59OhMnTsTX1xcRwdfXl9DQ0ELZtdVZ6BGFUipHjDHM2HiMD//cxaXkVJr4\nlMrReiIjI+nQoQOHDh1izpw5dOnSBUCDwYloUCilsu3Y+XjemrWNlfvO0NyvNB93b0St8p7ZXs+h\nQ4do164dZ86cYeHChbRp08YO1apbpUGhlMq26EspbD12kQ+63c7TLX1xyUETv127dtG+fXsuXbrE\nkiVLaN68uR0qVblBg0IpZZMDp2MJ33mKgffWokGVkqweFkiJHDbx27x5Mx07dsTV1ZXly5dr4z4n\np0GhlMpScmoaoSsO8vWSfXi4u9K9WTXKeRbNcUisXr2arl274u3tTXh4OHXq1MnlilVu06BQSmVq\ne+RFhs7cyo7j0XRtWIn3H/KnnGfOx3QIDw+nW7duVK1alfDwcH02Ip/QoFBKZehSUiq9fvgXN1cX\nxj/dlM7+lW9pfXPnzqVHjx7Uq1ePRYsWUamS7U9rK8fSoFBKXWN75EVur1KS4u6uhAQ1o0Hlknh7\nFLmldf7yyy/06tWLZs2a8ddff1GmjO1PayvH0wfulFIAxCam8O6c7Tzwv3+YtSkSgDtrlb3lkJgw\nYQJBQUG0bt2a8PBwDYl8SI8olFL8vSeK4b9v5/jFS/S724/O2Wjil5UxY8YwZMgQunTpwsyZMyle\nvHiurFflLQ0KpQq5TxbsZtzfB6hdwZMZg+6imW/pW16nMYYPPviAESNG8NhjjxEWFoa7e+ZjUCjn\npkGhVCGVmmZwdRFa1SyLm4vwQmBtirrd+kBAxhjeeOMNvvjiC/r27cuECRNwc9OPmvxM/+8pVchE\nRSfwzpzt1K3oxZCO9bi3bnnurVs+V9admprK4MGDmTBhAi+++CJfffUVLi56KTS/06BQqpAwxjB9\n4zE+nLeTxJS0bI0TYYvk5GT69u3L1KlTefvtt/nwww91DOsCQoNCqULg6DlLE79/9p+hhV8ZPu7e\nkJo5aOKXmYSEBHr27MncuXP5+OOPGTp0aK6tWzmeBoVShUBMQgrbj19k5MP+BLXwyVETv8zExsby\n8MMPs2TJEr799luCg4Nzbd3KOdj15KGIdBaRPSKyX0SGZTJPWxH5T0R2iMhye9ajVGGy71QMIX/v\nB7jSxK9Xq5x1es3MhQsX6NSpE8uWLWPKlCkaEgWU3Y4oRMQV+BboABwD1ovIXGPMznTzlAJCgM7G\nmAgRqWCvepQqLJJS0vhu+QH+t3Q/JYq68nhAdcp5FsXDPXd/3U+fPk2nTp3Yvn07v/32G927d8/V\n9SvnYc9TTy2A/caYgwAiMg3oBuxMN89TwCxjTASAMSbKjvUoVeBtPXaBN2dsZffJGB5sXIX3Hmxw\nS038MhMZGUn79u05cuQIf/zxB506dcr1bSjnYc+gqAocTff6GNDyunnqAkVE5G/AC/jaGPPj9SsS\nkQHAAEC7TSqVifikFHpPXEdRNxcm9A6gQ4OKdtnOwYMHad++PWfOnGHBggU6Kl0h4OiL2W5AM6Ad\nUBxYIyJrjTF7089kjAkFQgECAgJMnleplBPbHnmRBpVL4uHuxndPN6N+5ZJ4F7+1/kyZuTwqXUJC\ngo5KV4jYdDFbRGaJyP0ikp2L35FA9XSvq1nfS+8YsNAYE2eMOQOsABpnYxtKFVoxCcn83+xtPPC/\nf/h9s+VXq2XNsnYLiU2bNtGmTRvS0tJYvny5hkQhYusHfwiW6wn7RORjEalnwzLrgToiUkNE3IEn\ngLnXzTMHuEdE3ETEA8upqV021qRUobVsdxQdv1zB1H8jePaeGnRpaN+xHVatWsV9992Hh4cHK1eu\nxN/f367bU87FplNPxphwIFxEvIEnrd8fBSYAPxtjkjNYJkVEXgAWAq7ARGPMDhEZZJ0+3hizS0QW\nAFuBNOB7Y8z2XPnJlCqgPvprF98tP0idCp6EDL6LJj633sQvK5dHpatWrRrh4eFUr1795gupAkWM\nse2Uv4iUBZ4GegHHgTDgHqChMaatvQq8XkBAgNmwYUNebU4pp2CMIc2Aq4uwYu9pNhw5z/P31cqV\nJn5ZmTNnDo8//jj169dn0aJFVKxonwvkyv5EZKMxJiAny9p0RCEivwP1gJ+AB40xJ6yTfhUR/dRW\nyo5OXkzg/2Zvp34lL17vVI82dcvTJpea+GVl6tSp9O7dm4CAAObPn68DDhVitt71NNYYsyyjCTlN\nKKVU1owxTFt/lNF/7iIpNY07a5W16/bCwsIYPnw4ERERlC5dmnPnztG2bVvmzp2Ll5eXXbetnJut\nQdFARDYbYy4AiEhp4EljTIj9SlOq8Dp6Lp43Z2xlzcGztKpZho8fbYRfuRJ2215YWBgDBgwgPj4e\ngHPnzuHi4kLv3r01JJTNdz09dzkkAIwx54Hn7FOSUiouKYXdJ6MZ/UhDpj7byq4hAfDWW29dCYnL\n0tLSeP/99+26XZU/2HpE4SoiYqxXvq19nHRcQ6Vy0Z6TMYTvOsXz99WmfqWSrB7WjuLu9r1YffDg\nQcaPH8/Ro0cznB4REWHX7av8wdagWIDlwvV31tcDre8ppW5RUkoaIX/v59tl+/EqVoSezS1N/OwV\nEqmpqcyfP59x48axYMECXFxcKF68OJcuXbphXm2Zo8D2U09DgWXAYOvXEuBNexWlVGGx5egFHvzf\nP3wVvo+uDSuz+NU2dmniBxAVFcVHH31ErVq1eOihh9iyZQvvvvsuR44cYcKECXh4eFwzv4eHB6NG\njbJLLSp/sfWBuzRgnPVLKZUL4pNS6DNpHcXcXPm+dwDt7dDEzxjDqlWrCAkJYcaMGSQnJxMYGMgX\nX3zBQw89RJEilnYfQUFBAFfuevLx8WHUqFFX3leFm00P3IlIHeAjoAFQ7PL7xpia9istY/rAncrv\nth67gH8Vb1xchPWHz1Gvkhcli+Vuf6aYmBh+/vlnxo0bx7Zt2/D29qZv374MGjSI+vXr5+q2VP5g\n9wfugEnAe8CXwH1AP+w8Op5SBU10QjIfzd/NL+si+KJHY7o3q0Zzv9x9iG379u2MGzeOH3/8kdjY\nWJo0acKECRN48sknKVHCvndOqYLL1qAoboxZYr3z6QgwQkQ2Au/asTalCozwnacYPnsbp2MSGdCm\nJl0bVs61dSclJTFr1ixCQkJYuXIlRYsWpWfPngQHB9OiRQtEcm/oU1U42RoUidYW4/usjf4iAU/7\nlaVUwTF6/i5CVxykfiUvQnsF0Lh6qVxZb0REBN999x3ff/89UVFR1KxZk88++4x+/fpRtqx9n+JW\nhYutQfEy4AG8BIzEcvqpj72KUiq/M8aQmmZwc3WhdZ1yeBZ1Y9C9tXB3u7UztmlpaSxatIhx48Yx\nb948AB544AGCg4Pp0KEDLi56RljlvpsGhfXhup7GmNeBWCzXJ5RSmThx8RL/9/t26lf24o1O9Wld\npzyt69xaE7+zZ88yadIkxo8fz4EDB6hQoQLDhg1jwIAB+Pr65lLlSmXspkFhjEkVkXvyohil8rO0\nNMMv6yP4aP5uUtMMreuUu6X1GWNYt24dISEh/PrrryQmJtK6dWs+/PBDHn30UdzdtTmCyhu2nnra\nLCJzgelA3OU3jTGz7FKVUvlMxNl43pixhX8PnePu2mX56JFG+JT1uPmCXNu11cfHh3fffZe0tDTG\njRvHpk2b8PT05JlnnmHw4ME6spxyCFuDohhwFghM954BNCiUAuKTU9gfFcsn3RvyeEB1m+80ur5r\n65EjR3jmmWcAaNiwISEhITz99NPawVU5lM0j3DkLfeBOOYvdJ6NZvOMUL7arA0BCcirFimSvP5Of\nnx9Hjhy54f2KFSty4sQJvbVV5Zq8GOFuEpYjiGsYY/rnZKNK5WeJKal8u3Q/IX8fwLt4EZ5s6UM5\nz6LZDonExMQMQwIsfZk0JJSzsPXU07x03xcDHsEybrZShcqmiPMMnbGVfVGxPNqkKu880IDSJbJ/\nUXnp0qUEBwdnOl27tipnYtNN18aYmem+woDHAR0CVRUq8Ukp9J+8nrjEFCb1a86YnndkOyROnTrF\n008/Tbt27UhOTuaNN97Qrq3K6dl6RHG9OkCF3CxEKWe1OeI8jauVwsPdjR/6BFCvUkk8i2bvVyc1\nNZXQ0FDeeustLl26xDvvvMNbb71F8eLFady4sXZtVU7N1u6xMVx7jeIk8JYxZqa9CsuMXsxWeeXi\npWRG/7mLXzccvdLELyc2bdrEoEGDWL9+PYGBgYSEhFCvXr1crlaprNn9YrYxRu/NU4XKwh0neWf2\nds7GJTHo3lrc3yj7Tfyio6N55513+OabbyhfvjxhYWE8+eSTepFa5Tu23vX0CLDUGHPR+roU0NYY\nM9uexSnlCCPn7eSHfw5xW+WS/NCnOQ2reWdreWMM06dP55VXXuHkyZMMHjyYUaNGUapU7jQDVCqv\n2Xqi9T0MjJrnAAAbnElEQVRjzO+XXxhjLojIe4AGhSoQ0jfxu69eBUp7FGHgvbUo4pq9Jnv79+/n\n+eefZ9GiRTRt2pQ5c+bQvHlzO1WtVN6w9bcgo/lyeiFcKacSeeES/Sav58vwvQDcU6ccLwTWyVZI\nJCQk8P777+Pv78+aNWsYO3Ys69at05BQBYKtH/YbRGQM8K319fPARvuUpFTeSEszhP17hI//2k2a\ngcD6ObuRLzw8nODgYPbt20fPnj0ZM2YMVapUyeVqlXIcW4PiReAd4Fcsdz8txhIWSuVLh8/E8eaM\nraw7fI7Wdcox+pGGVC9jWxO/y06cOMFrr73GtGnTqF27NosWLaJDhw52qlgpx7H1rqc4YJida1Eq\nzySmpHHwTByfPdaIx5pVy9adSKmpqYwbN47hw4eTkJDAiBEjGDp0KMWKFbNjxUo5jk0nYUVksfVO\np8uvS4vIQvuVpVTu23H8Il9Zr0PUq+TFP0Pvo0c2Or0CbNiwgZYtW/Liiy/SsmVLtm/fznvvvach\noQo0W6/WlTPGXLj8whhzHn0yW+UTCcmpfLZwNw99s4qf10ZwJjYRIFtN/C5evMgLL7xAixYtiIyM\nZNq0aSxcuJA6derYq2ylnIat1yjSRMTHGBMBICJ+ZNBNVilns/HIOd6csZUDp+Po3rQa7zxwG6U8\nbO/PZIxh2rRpvPbaa0RFRfHCCy8wcuRIvL2z92yFUvmZrUExHPhHRJYDArQGBtitKqVyQXxSCs9M\n2UAJdzem9G/BvXWzN2713r17CQ4OZsmSJQQEBDBv3jyaNWtmp2qVcl62XsxeICIBWMJhM5YH7S7Z\nszClcmrjkfM0qX65iV9z6lXyylYTv4SEBD766CM+/vhjihcvzrfffsvAgQNxdc3eeBNKFRS2Xsx+\nFlgCDAFeB34CRtiwXGcR2SMi+0Uk07umRKS5iKSIyGO2la3UjS7GJ/PG9C10H7eaWZsjAWjmWzrL\nkAgLC8PPzw8XFxf8/PwYOnQo/v7+fPDBBzz22GPs3r2b4OBgDQlVqNn6Z9bLQHNgrTHmPhGpD4zO\nagERccXygF4H4BiwXkTmGmN2ZjDfJ8Ci7Bav1GULtp/gnTk7OBeXRHDbWjxgQxO/jMar/vTTT6lU\nqRLh4eG0a9fO3mUrlS/YGhQJxpgEEUFEihpjdovIzfoktwD2G2MOAojINKAbsPO6+V4EZmIJIqWy\n7YM/djJx1SEaVC7JpL7N8a9q24Xm4cOHXwmJ9Nzd3TUklErH1qA4Zn2OYjawWETOAxkP9ntVVeBo\n+nUALdPPICJVsQyreh9ZBIWIDMB68VyHiFRwbRO/drdVoKynOwPa1LSpP1NKSgp//vlnpuNVHz16\nNMP3lSqsbL2Y/Yj12xEisgzwBhbkwva/AoYaY9KyeujJGBMKhIJl4KJc2K7Kx46ei+ft37fhX9Wb\noZ3rc3ftctxdu9xNlzt16hQ//PAD48eP5+jRo7i6upKamnrDfPrHiFLXynYHWGPMchtnjQSqp3td\nzfpeegHANGtIlAO6ikiKjnOhMpKWZvhxzWE+XbgHATrdXummyxhjWLVqFSEhIcyYMYPk5GTatWvH\n119/TWxsLIMGDbrm9JOOV63UjezZKnw9UEdEamAJiCeAp9LPYIypcfl7EZkMzNOQUBk5dCaON6Zv\nYcOR89xbtzyjHvGnWunMm/jFxsYSFhZGSEgIW7duxdvbm+DgYAYNGkT9+vWvzOfi4qLjVSt1E3YL\nCmNMioi8ACwEXIGJxpgdIjLIOn28vbatCp7k1DSOnItnzOONeaRJ1Uz7M+3atYuQkBCmTJlCTEwM\njRs3JjQ0lKeeeooSJUrcMH9QUJAGg1I3Icbkr1P+AQEBZsOGDY4uQ+WB7ZEXWbzzFK92qAtAYkoq\nRd1ufJ4hOTmZOXPmEBISwrJly3B3d+fxxx8nODiYVq1a6RjVSgEistEYE5CTZXWUOuV0EpJT+XrJ\nPkJXHKRMCXd63+lLWc+iN4TE8ePHmTBhAqGhoRw/fhxfX18++ugj+vfvT4UK2rNSqdyiQaGcyvrD\n5xg6YysHz8TRo1k1/u/+Bnh7FLky3RjD33//TUhICL///jupqal07tyZ8ePH07VrV32CWik70KBQ\nTiMuMYXnftyAZ1E3fnqmBa3rXG3iFx0dzY8//khISAi7du2iTJkyvPrqqwwcOJDatWs7sGqlCj4N\nCuVw6w+fo5lPaUoUdWNi3+bUq+hFCWt/pm3bthESEsJPP/1EXFwcAQEBTJo0iZ49e1K8eHEHV65U\n4aBBoRzmfFwSI+ftZNbmSD7v0ZjHmlWjqU9pkpKS+OWXXwgJCeGff/6hWLFiPPHEEwQHB9O8uXZ6\nUSqvaVCoPGeMYf62k7z52wbikgwX107n9d9e5vgrLxEVFcWECROIioqiVq1afP755/Tt25eyZcs6\numylCi0NCpXnPpi3k0mrDpN86hCn//yK5NOHuAC8/PLLADz44IMEBwfTsWNHXFxsHa1XKWUvGhQq\nTxhjSEkzFHF1ocNtFZn07ZccXzQJTNo181WtWpW5c+c6qEqlVEb0zzVld0fPxdPrh3V8sWgvAHfW\nKsuRhT/cEBJgeTZCKeVcNCiU3aSmGSb+c4iOX67gv6MX8CnjwbFjx+jSpUumy2jnVqWcjwaFsouD\np2PpMX41H8zbScuaZVj4SmuSdy/D39+flStX0qdPHzw8rm3qp51blXJOGhTKLlLTDJEXLvFVzzv4\nqHN1Bvd5gr59+9KwYUO2bNnC5MmTCQ0NxdfXFxHB19eX0NBQbdCnlBPSpoAq12w9doHFO08xpKNl\nlNzElFRmz5xBcHAw8fHxjB49mpdeeknbbCjlANoUUDlUQnIqXy7ey4SVBynvVZS+d/mRdima559/\nnunTp9OiRQumTJlyzTgQSqn8Q4NC3ZK1B88ybOZWDp+N58kW1RnW5TaWLfyTgQMHcv78eUaPHs0b\nb7yBm5v+U1Mqv9LfXpVjcYkpDPp5IyWLFWHqsy25rawrLwzoz88//0yTJk0IDw+nYcOGji5TKXWL\nNChUtq07dI4AX0sTv8n9WlC3oifLlyzG/95niYqK4r333mP48OEUKVLk5itTSjk9vetJ2excXBKv\nTNvM49+tYdbmSABqervwUvAgunbtSunSpfn3338ZMWKEhoRSBYgeUaibMsYwb+sJRszdwcVLybzc\nrg4PNq7MkiVL6N+/P8eOHWPYsGGMGDGCokWLOrpcpVQu06BQN/X+HzuZvPowjat5E/ZcS6p5uvDa\nyy8REhJC3bp1WbVqFa1atXJ0mUopO9GgUBkyxpCcanB3c6Hj7RWpWqo4/e+pwepV/3B/374cOnSI\nV199lQ8//PCGJ6yVUgWLXqNQNzhyNo6nJvzLF4v2AHBXrXI83bwyb77xOvfeey8Af//9N2PGjNGQ\nUKoQ0CMKdUVqmmHSqkN8vmgPRVxceLhJFQD+/fdf+vTpw549ewgODuaTTz7B09PTwdUqpfKKBoUC\nYH9ULEOmb2HL0Qu0v60CHz7ckNLFhLfffptPPvmEqlWrsnjxYtq3b+/oUpVSeUyDQgGWaxJR0QmM\nfbIJDzaqzH///UfHPn3Ytm0b/fv3Z8yYMXh7ezu6TKWUA+g1ikLsv6MX+GzhbgDqVPRi+Rv30aVB\neT744ANatGjBmTNnmDdvHj/88IOGhFKFmB5RFEKXklIZs3gPP/xziApexeh/dw3KehZl7+6d9OnT\nh02bNhEUFMTYsWMpU6aMo8tVSjmYBkUhs/rAGYbN3EbEuXjYv5L1876h6XdladWqFbNnz8bb25tZ\ns2bxyCOPOLpUpZST0KAoROISU3g+bBOSksCFWSO5uM8yrkdERBwRERG0aNGCefPmUb58eQdXqpRy\nJnqNohBYc+AsaWnmShO/c2FDroREeidPntSQUErdQIOiADsbm8iLv2zmyQlr+d3axK9x9VJEHDqQ\n4fxHjx7Ny/KUUvmEnnoqgIwxzN1ynBFzdxCXmMqQDnV5sHEVLl26xMiRI8ls+FsfH588rlQplR9o\nUBRA783dwY9rjtDEpxSfdm9EnYpeLFmyhIEDB3LgwAHatGnD+vXruXTp0pVlPDw8GDVqlAOrVko5\nKz31VECkpRmSUtIA6OJfmXceaMCMQXdR2jWRPn360L59e0SEJUuWsHz5ciZMmICvry8igq+vL6Gh\noQQFBTn4p1BKOSPJ7DRErqxcpDPwNeAKfG+M+fi66UHAUECAGGCwMWZLVusMCAgwGzbceCG2MDt0\nJo5hM7fSuHop3u56G2A5/RQWFsarr77KhQsXGDp0KMOHD6d48eIOrlYp5QgistEYE5CTZe126klE\nXIFvgQ7AMWC9iMw1xuxMN9sh4F5jzHkR6QKEAi3tVVNBk5KaxsRVh/hi0V7c3Vzo3rQaAAcOHGDw\n4MEsXryYO++8k9DQUPz9/R1crVIqv7LnNYoWwH5jzEEAEZkGdAOuBIUxZnW6+dcC1exYT4GyPyqG\n137bwtZjF+nQoCIfPuxPmeKufPLJJ4wYMQJ3d3dCQkIYOHAgLi56hlEplXP2DIqqQPr7LY+R9dHC\nM8BfGU0QkQHAANA7c9I7E5PIN0814f6GlVm/fj0dn3uOrVu38uijjzJ27FiqVq3q6BKVUgWAU/yp\nKSL3YQmKoRlNN8aEGmMCjDEBhfmBsE0R5/lkgaWJX+0KXix/8z7ureHFK6+8QqtWrTh79iy///47\nM2fO1JBQSuUaex5RRALV072uZn3vGiLSCPge6GKMOWvHevKt+KQUPl+4l0mrD1G5ZDGevcfSxG/B\n/D8JDg4mMjKS559/nlGjRlGyZElHl6uUKmDsGRTrgToiUgNLQDwBPJV+BhHxAWYBvYwxe+1YS771\nz74zDJu1lWPnL9H7Tl/e7FyfmHOn6dHvJWbMmIG/vz/Tp0+nVatWji5VKVVA2S0ojDEpIvICsBDL\n7bETjTE7RGSQdfp44F2gLBAiIgApOb19qyCKS0zhxV82UcrDnd8G3kmAbylCQ0MZNmwYCQkJjB49\nmtdff50iRYo4ulSlVAFm1+co7KEwPEexev8ZWtYsi6uLsO3YRepU9OTgvj0MGDCAVatWERgYyPjx\n46lTp46jS1VK5RO38hyFU1zMVhanYxJ5PmwTT33/75UmfnXKFWX0yPe544472L17N1OmTCE8PFxD\nQimVZ7TXkxMwxvD75kg+mLeT+MRU3uhUj253VGH58uUMGDCAvXv30qtXL7744gttA66UynMaFE7g\nnTnb+XltBE19SvHpY40o45bM4IED+OGHH6hZsyaLFi2iQ4cOji5TKVVIaVA4SFqaITktjaJurjzQ\nqAq1y3vydCtfZkz/jZdffpmzZ88ydOhQ3n33XTw8PBxdrlKqENOgcIADp2MZNnMrRWNP8M83Q4iI\niKBKlSp8WbYsW7dupXnz5ixatIjGjRs7ulSllNKgyEvJqWlMWHmQr8L34ZKWQtTCHzh/5AgAkZGR\nREZG0qtXLyZNmoSrq6uDq1VKKQsNijyy91QMr/76HzuOR9P59kosGNWP83u23TDfihUrNCSUUk5F\ngyKPuIhwIT6ZcUFN6dKwMi59tmc4X0RERB5XppRSWdPnKOxo45FzfPTXLgBqV/Bk+Rtt6XBbeYYO\nHarjViul8g0NCjuIS0xhxNwdPDZ+DfO2nOBcXBIAp06eIDAwkE8//ZR27drdcDeTjlutlHJGGhS5\nbMXe03T8cgVT1hymz51+LHq1DWVKuLN48WKaNGnCpk2bCAsLIzw8nNDQUB23Winl9LTXUy6KS0yh\n9afLKOVRhE+7NyLArwypqal88MEHjBw5kgYNGjBjxgzq16/v6FKVUoWMU46ZXZis3Heau2qVo0RR\nN37s34LaFTwpVsSVU6dOERQUxJIlS+jTpw8hISH68JxSKt/RU0+3ICo6gUE/baTXD+uYbW3i51/V\nm2JFXFmxYgVNmjRh1apVTJw4kcmTJ2tIKKXyJQ2KHDDGMH3DUdqPWc7SPVEM7VyfbndUASAtLY2P\nP/6Y++67Dy8vL/7991/69evn4IqVUirn9NRTDgyfvZ2p/0bQ3K80H3dvRK3yngCcPXuW3r17M3/+\nfJ544glCQ0Px8vJycLVKKXVrNChslL6JX7fGVbitkhdBLX1xcREA1q5dy+OPP86pU6cICQlh0KBB\nWEftU0qpfE1PPdlgf1QMPb5bw2cL9gDQsmZZet3ph4uLYIzhq6++onXr1ri5ubF69WoGDx6sIaGU\nKjA0KLKQnJrGt8v20/XrfzhwOpbbq5a8ZvqFCxfo3r07r776Kg888ACbNm2iWbNmDqpWKaXsQ089\nZWLvqRhemfYfO09Ec3/Dyox46HbKexW9Mn3Tpk306NGDiIgIvvjiC1599VU9ilBKFUgaFJlwdRFi\nEpMZ/3QzOvtXuvK+MYbvvvuOl19+mQoVKrB8+XLuuusuB1aqlFL2paee0ll36Byj/twJQK3yniwb\n0vaakIiJiSEoKIjBgwcTGBjI5s2bNSSUUgWeBgUQm5jCO7O38/h3a1iw4+SVJn5urld3z7Zt22je\nvDm//voro0eP5s8//6RcuXKOKlkppfJMoT/1tGxPFMNnbeNEdAL9767B653q4uHuRlhYGMOHDyci\nIoIyZcoQHR1N2bJlWbJkCW3btnV02UoplWcKdVDEJqYw5LctlC3hzszBd9HUpzQAYWFhDBgwgPj4\neMDyIJ2LiwvvvPOOhoRSqtApdN1jjTEs33ua1nXK4+oi7DweTa0KJSjqdnX4UT8/P45Yx7JOz9fX\nl8OHD+d420op5Si30j22UF2jiIpOYOBPG+k7af2VJn4NqpS8JiQg8+FIdZhSpVRhVChOPVma+B1j\n5J87SUpJ460uV5v4ZcTHxyfDIwodplQpVRgViiOKt3/fzpszt3Jb5ZIseKUNA++tdc0dTdcbNWqU\nDlOqlFJWBfaIIjXNkJyaRrEirjzSpCq3VynJUy18rjTxy8rl4Ugv3/Xk4+PDqFGjdJhSpVShVCAv\nZu89FcObM7bSzLc07zzQII8qU0op56UXs62SUtIYu2Qf949dyZGzcTSq5u3okpRSKt8rMKeedp+M\n5pVp/7H7ZAwPNq7CiAcbUNaz6M0XVEoplaUCExRFXF24lJzKhN4BdGhQ0dHlKKVUgZGvTz2tPXiW\nD+ddbeK3dEhbDQmllMpldg0KEeksIntEZL+IDMtguojIWOv0rSLS1Jb1xiQkM/z3bTwRupZFO09d\naeLnasMdTUoppbLHbqeeRMQV+BboABwD1ovIXGPMznSzdQHqWL9aAuOs/81UTEIyHb9cwanoBJ69\npwZDOtajuLtrVosopZS6Bfa8RtEC2G+MOQggItOAbkD6oOgG/Ggs9+iuFZFSIlLZGHMis5UePX8J\n32JuhATdRRNrEz+llFL2Y8+gqAocTff6GDceLWQ0T1XgmqAQkQHAAOvLxMWvtd2++LXcLTafKgec\ncXQRTkL3xVW6L67SfXFVvZwumC/uejLGhAKhACKyIacPjRQ0ui+u0n1xle6Lq3RfXCUiOW67bc+L\n2ZFA9XSvq1nfy+48SimlHMieQbEeqCMiNUTEHXgCmHvdPHOB3ta7n1oBF7O6PqGUUirv2e3UkzEm\nRUReABYCrsBEY8wOERlknT4emA90BfYD8UA/G1YdaqeS8yPdF1fpvrhK98VVui+uyvG+yHdNAZVS\nSuWtfP1ktlJKKfvToFBKKZUlpw0Ke7X/yI9s2BdB1n2wTURWi0hjR9SZF262L9LN11xEUkTksbys\nLy/Zsi9EpK2I/CciO0RkeV7XmFds+B3xFpE/RGSLdV/Ycj003xGRiSISJSLbM5mes89NY4zTfWG5\n+H0AqAm4A1uABtfN0xX4CxCgFfCvo+t24L64Cyht/b5LYd4X6eZbiuVmicccXbcD/12UwtIJwcf6\nuoKj63bgvngb+MT6fXngHODu6NrtsC/aAE2B7ZlMz9HnprMeUVxp/2GMSQIut/9I70r7D2PMWqCU\niFTO60LzwE33hTFmtTHmvPXlWizPoxREtvy7AHgRmAlE5WVxecyWffEUMMsYEwFgjCmo+8OWfWEA\nLxERwBNLUKTkbZn2Z4xZgeVny0yOPjedNSgya+2R3XkKguz+nM9g+YuhILrpvhCRqsAjWBpMFmS2\n/LuoC5QWkb9FZKOI9M6z6vKWLfviG+A24DiwDXjZGJOWN+U5lRx9buaLFh7KNiJyH5aguMfRtTjQ\nV8BQY0ya5Y/HQs0NaAa0A4oDa0RkrTFmr2PLcohOwH9AIFALWCwiK40x0Y4tK39w1qDQ9h9X2fRz\nikgj4HugizHmbB7Vltds2RcBwDRrSJQDuopIijFmdt6UmGds2RfHgLPGmDggTkRWAI2BghYUtuyL\nfsDHxnKifr+IHALqA+vypkSnkaPPTWc99aTtP6666b4QER9gFtCrgP+1eNN9YYypYYzxM8b4ATOA\n4AIYEmDb78gc4B4RcRMRDyzdm3flcZ15wZZ9EYHlyAoRqYilk+rBPK3SOeToc9MpjyiM/dp/5Ds2\n7ot3gbJAiPUv6RRTADtm2rgvCgVb9oUxZpeILAC2AmnA98aYDG+bzM9s/HcxEpgsItuw3PEz1BhT\n4NqPi8gvQFugnIgcA94DisCtfW5qCw+llFJZctZTT0oppZyEBoVSSqksaVAopZTKkgaFUkqpLGlQ\nKKWUypJT3h6rlDMSkc+w3Fo4H0sTunhjzI/XzeMHzDPG+Od5gUrZiQaFUrYbAJQxxqQ6uhCl8pKe\nelKFgoj0tvbf3yIiP4mIn4gstb63xPp0OyIy2dqvf7WIHLw8noWIzMXSdXSjiPQUkREi8rp1WjPr\nercAz6fbpquIfCYi663bGWh9v621Ud8MEdktImHWrqaXx9FYbV3fOhHxymw9SuUVDQpV4InI7cD/\nAYHGmMbAy8D/gCnGmEZAGDA23SKVsTRWfAD4GMAY8xBwyRhzhzHm1+s2MQl40bru9J7B0iKhOdAc\neE5EalinNQFeARpgGUfhbmv7iV+xdDZtDLQHLt1kPUrZnZ56UoVBIDD9cssGY8w5EbkTeNQ6/Sfg\n03Tzz7a2oN5p7QuUKREpBZSyjgNweV1drN93BBrJ1VH2vIE6QBKwzhhzzLqO/wA/4CJwwhiz3lpn\ntHV6Zus5lK29oFQOaVAodaPEdN/fSq9ywXKksfCaN0XaXreNVLL+XcxwPUrlFT31pAqDpUAPESkL\nICJlgNVYuowCBAErc7JiY8wF4IKIXB4DJCjd5IXAYBEpYt1uXREpkcXq9gCVRaS5dX4vEXHLwXqU\nylV6RKEKPGsn0VHAchFJBTZjGS51koi8AZzm1roP9wMmiogBFqV7/3ssp5Q2WS9WnwYezqLOJBHp\nCfxPRIpjuT7RPrvrUSq3afdYpZRSWdJTT0oppbKkQaGUUipLGhRKKaWypEGhlFIqSxoUSimlsqRB\noZRSKksaFEoppbL0/9GWL1+jlnFjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe2847ed68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_calibration(val_true_targets, val_predictions, n_bins=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### per class accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEKCAYAAAACS67iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEY9JREFUeJzt3XuQnXV9x/H3R8DxLqGsMVXpyogiUontmvFWBy9UhHGQ\n6lixVapOYzuW6ox2zDhVcewltl46UxUnCJI6VktH8Aba0ohlrAoGDCEIrVSigpGsghdsSw18+8d5\nIsc0yzm7ey6bn+/XzM6e85znnOeTk80nz/6e5/mdVBWSpAPfvaYdQJI0Gha6JDXCQpekRljoktQI\nC12SGmGhS1IjLHRJaoSFLkmNsNAlqREHT3Jjhx9+eM3Ozk5yk5J0wLvyyiu/V1Uzg9abaKHPzs6y\ndevWSW5Skg54Sb45zHoOuUhSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMm\neqWotFLNbrhoatveufHkqW1bbXEPXZIaMbDQk9wnyRVJrk5ybZK3dsvPTHJzkm3d10njjytJWsgw\nQy53AM+sqtuTHAJ8IclnusfeXVXvGF88SdKwBhZ6VRVwe3f3kO6rxhlKkrR4Q42hJzkoyTZgN3BJ\nVV3ePXRGku1Jzk2yaoHnrk+yNcnW+fn5EcWWJO1rqEKvqjurai3wcGBdkmOBs4AjgbXALuCdCzx3\nU1XNVdXczMzA+dklSUu0qLNcquoHwKXAiVV1S1f0dwFnA+vGEVCSNJxhznKZSXJod/u+wAnA9UnW\n9K12KrBjPBElScMY5iyXNcDmJAfR+w/g/Kr6dJIPJVlL7wDpTuBV44spSRpkmLNctgNP2M/yl44l\nkSRpSbxSVJIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSF\nLkmNGGa2Rf2Cmd1w0dS2vXPjyVPbtnSgcw9dkhphoUtSIyx0SWqEhS5JjfCgqFaUaR6QlQ507qFL\nUiMsdElqxMBCT3KfJFckuTrJtUne2i0/LMklSb7efV81/riSpIUMs4d+B/DMqjoOWAucmORJwAZg\nS1UdBWzp7kuSpmRgoVfP7d3dQ7qvAk4BNnfLNwPPH0tCSdJQhjrLJclBwJXAo4D3VtXlSVZX1a5u\nle8Cqxd47npgPcARRxyx/MRSY6Z1Zo/TLLRnqIOiVXVnVa0FHg6sS3LsPo8Xvb32/T13U1XNVdXc\nzMzMsgNLkvZvUWe5VNUPgEuBE4FbkqwB6L7vHn08SdKwhjnLZSbJod3t+wInANcDnwRO71Y7HfjE\nuEJKkgYbZgx9DbC5G0e/F3B+VX06yZeA85O8Evgm8KIx5pQkDTCw0KtqO/CE/Sz/PvCscYSSJC2e\nV4pKUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREW\nuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjRhY6EkekeTSJF9Lcm2S13TLz0xyc5Jt3ddJ\n448rSVrIwA+JBvYAr6uqq5I8ELgyySXdY++uqneML54kaVgDC72qdgG7uts/TnId8LBxB5MkLc6i\nxtCTzAJPAC7vFp2RZHuSc5OsGnE2SdIiDF3oSR4AfAx4bVX9CDgLOBJYS28P/p0LPG99kq1Jts7P\nz48gsiRpf4Yq9CSH0CvzD1fVBQBVdUtV3VlVdwFnA+v299yq2lRVc1U1NzMzM6rckqR9DHOWS4Bz\ngOuq6l19y9f0rXYqsGP08SRJwxrmLJenAi8FrkmyrVv2RuC0JGuBAnYCrxpLQknSUIY5y+ULQPbz\n0MWjjyNJWiqvFJWkRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtd\nkhphoUtSI4aZbVFSg2Y3XDS1be/cePLUtt0y99AlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0\nSWrEwEJP8ogklyb5WpJrk7ymW35YkkuSfL37vmr8cSVJCxlmD30P8LqqOgZ4EvDqJMcAG4AtVXUU\nsKW7L0makoGFXlW7quqq7vaPgeuAhwGnAJu71TYDzx9XSEnSYIsaQ08yCzwBuBxYXVW7uoe+C6we\naTJJ0qIMXehJHgB8DHhtVf2o/7GqKqAWeN76JFuTbJ2fn19WWEnSwoYq9CSH0CvzD1fVBd3iW5Ks\n6R5fA+ze33OralNVzVXV3MzMzCgyS5L2Y5izXAKcA1xXVe/qe+iTwOnd7dOBT4w+niRpWMNMn/tU\n4KXANUm2dcveCGwEzk/ySuCbwIvGE1GSNIyBhV5VXwCywMPPGm0cSdJSeaWoJDXCQpekRljoktQI\nC12SGmGhS1IjLHRJasQw56FrSmY3XDTtCJIOIO6hS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY\n6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNWJgoSc5N8nuJDv6lp2Z5OYk27qvk8YbU5I0\nyDB76OcBJ+5n+buram33dfFoY0mSFmtgoVfVZcCtE8giSVqG5Yyhn5Fkezcks2pkiSRJS7LUQj8L\nOBJYC+wC3rnQiknWJ9maZOv8/PwSNydJGmRJhV5Vt1TVnVV1F3A2sO4e1t1UVXNVNTczM7PUnJKk\nAZZU6EnW9N09Fdix0LqSpMkY+CHRST4CHA8cnuQm4C3A8UnWAgXsBF41xoySpCEMLPSqOm0/i88Z\nQxZJ0jJ4pagkNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqxMDTFgWzGy6adgRJGsg9dElqhIUuSY2w\n0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEYMLPQk5ybZ\nnWRH37LDklyS5Ovd91XjjSlJGmSYPfTzgBP3WbYB2FJVRwFbuvuSpCkaWOhVdRlw6z6LTwE2d7c3\nA88fcS5J0iItdQx9dVXt6m5/F1i90IpJ1ifZmmTr/Pz8EjcnSRpk2QdFq6qAuofHN1XVXFXNzczM\nLHdzkqQFLLXQb0myBqD7vnt0kSRJS7HUQv8kcHp3+3TgE6OJI0laqmFOW/wI8CXgMUluSvJKYCNw\nQpKvA8/u7kuSpujgQStU1WkLPPSsEWeRJC2DV4pKUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRgw8\nbVGSRm12w0VT2e7OjSdPZbuT4h66JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREW\nuiQ1wkKXpEYcMJf+T+tSYUk6ULiHLkmNWNYeepKdwI+BO4E9VTU3ilCSpMUbxZDLM6rqeyN4HUnS\nMjjkIkmNWG6hF/AvSa5Msn4UgSRJS7PcIZenVdXNSR4CXJLk+qq6rH+FrujXAxxxxBHL3JwkaSHL\n2kOvqpu777uBC4F1+1lnU1XNVdXczMzMcjYnSboHSy70JPdP8sC9t4HfBHaMKpgkaXGWM+SyGrgw\nyd7X+fuq+uxIUkmSFm3JhV5V3wCOG2EWSdIyeNqiJDXigJnLRZKWa5pzQu3cePLYt+EeuiQ1wkKX\npEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElq\nhIUuSY2w0CWpERa6JDXCQpekRiyr0JOcmOTfk9yQZMOoQkmSFm/JhZ7kIOC9wHOBY4DTkhwzqmCS\npMVZzh76OuCGqvpGVf0v8FHglNHEkiQt1nIK/WHAt/vu39QtkyRNwcHj3kCS9cD67u4dSXaMe5tL\ndDjwvWmHWIDZlsZsS2O2pbnHbHn7sl77V4ZZaTmFfjPwiL77D++W/Zyq2gRsAkiytarmlrHNsTHb\n0phtacy2NGa7Z8sZcvkKcFSSRya5N/Bi4JOjiSVJWqwl76FX1Z4kfwT8E3AQcG5VXTuyZJKkRVnW\nGHpVXQxcvIinbFrO9sbMbEtjtqUx29KY7R6kqqadQZI0Al76L0mNGHmhD5oOIMnRSb6U5I4krx/1\n9peZ7XeSbE9yTZIvJjluBWU7pcu2LcnWJE9bKdn61ntikj1JXrhSsiU5PskPu/dtW5I3r5Rsffm2\nJbk2yb+ulGxJ/qTvPduR5M4kh62QbA9O8qkkV3fv28snkWvIbKuSXNj9W70iybGTygZAVY3si97B\n0f8EjgTuDVwNHLPPOg8Bngj8OfD6UW5/BNmeAqzqbj8XuHwFZXsAdw+RPR64fqVk61vvc/SOqbxw\npWQDjgc+Pamfs0VmOxT4GnBEd/8hKyXbPus/D/jcSskGvBF4e3d7BrgVuPcKyfbXwFu620cDWyb5\nczfqPfSB0wFU1e6q+grw0xFvexTZvlhVt3V3v0zv3PqVku326n5KgPsDkzr4MewUD2cAHwN2TyjX\nYrJNwzDZXgJcUFXfgt6/jRWUrd9pwEcmkmy4bAU8MEno7ejcCuxZIdmOobdjQ1VdD8wmWT2BbMDo\nh1xW8nQAi832SuAzY010t6GyJTk1yfXARcArVkq2JA8DTgXOmlCmvYb9O31K9yvwZ5I8bjLRhsr2\naGBVks8nuTLJy1ZQNgCS3A84kd5/1pMwTLb3AI8FvgNcA7ymqu5aIdmuBn4LIMk6eld4TmrH0IOi\n+5PkGfQK/Q3TztKvqi6sqqOB5wNvm3aePn8DvGFC/6gW6yp6QxqPB/4W+PiU8/Q7GPh14GTgOcCb\nkjx6upH+n+cB/1ZVt047SJ/nANuAXwbWAu9J8qDpRvqZjcChSbbR+631q8Cdk9r4qOdyGWo6gCkZ\nKluSxwMfAJ5bVd9fSdn2qqrLkhyZ5PCqGve8FsNkmwM+2vsNmMOBk5Lsqapxl+fAbFX1o77bFyd5\n3wp6324Cvl9VPwF+kuQy4DjgP1ZAtr1ezOSGW2C4bC8HNnZDkDckuZHeePUV087W/by9HKAbEroR\n+MaYc/1cgFEeNDi4C/9I7j5o8LgF1j2TyR4UHZgNOAK4AXjKJA9kDJntUdx9UPTX6P0gZSVk22f9\n85jcQdFh3reH9r1v64BvrZT3jd6wwZZu3fsBO4BjV0K2br0H0xufvv8k/j4X8b6dBZzZ3V7d/Vs4\nfIVkO5TuAC3w+8DfTeq9q6rR7qHXAtMBJPmD7vH3J3kosBV4EHBXktfSO1L8owVfeELZgDcDvwS8\nr9vb3FMTmGxnyGwvAF6W5KfAfwO/Xd1PzQrINhVDZnsh8IdJ9tB73168Ut63qrouyWeB7cBdwAeq\nauyzkS7i7/RU4J+r9xvERAyZ7W3AeUmuAUJvuG/sMzAOme2xwOYkBVxLb+h2YrxSVJIa4UFRSWqE\nhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXdqPJKO+iloaOwtdB5wkH+8ms7o2yfpu2YlJrurmyN7S\nLXtAkg+mN7/99iQv6Jbf3vdaL0xyXnf7vCTvT3I58FdJ1qU3d/9X05sf/zHdegcleUc3T/j2JGck\neWaSj/e97glJLpzcuyKNfi4XaRJeUVW3Jrkv8JUknwDOBp5eVTf2fRDDm4AfVtWvQu/DB4Z47YfT\nm/rhzm7Cp9/orhB8NvAX9K7YXQ/MAmu7xw4DbqN3hfFMVc3Tm8/j3NH9kaXBLHQdiP44yand7UfQ\nK9jLqupGgLp7ZsBn05tcim75bQz2j1W1d3a8B9O7jPsoenNwH9L3uu+vqj3920vyIeB3k3wQeDIw\nqelwJcBC1wEmyfH0CvXJVfVfST5PbyrVoxfxMv3zXdxnn8f65y15G3BpVZ2aZBb4/IDX/SDwKeB/\n6P3HMIkPXZB+xjF0HWgeDNzWlfnRwJPolfLTkzwSoG/I5RLg1Xuf2DfkckuSxya5F70JqO5pW3un\nR/29vuWXAK/ae+B07/aq6jv0PnThT+mVuzRRFroONJ8FDk5yHb0PE/gyME9v2OWCJFcD/9Ct+2f0\nPhFoR7f8Gd3yDcCngS8Cu+5hW38F/GWSr/Lzv81+gN40vNu7131J32MfBr5dVdct488oLYmzLUoj\nlOQ9wFer6pxpZ9EvHgtdGpEkV9Ibgz+hqu6Ydh794rHQJakRjqFLUiMsdElqhIUuSY2w0CWpERa6\nJDXCQpekRvwfTAQO1QA44xQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe24dc1358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "per_class_acc = per_class_accuracy(val_true_targets, val_predictions)\n",
    "plt.hist(per_class_acc);\n",
    "plt.xlabel('accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "syringe 0.14\n",
      "Labrador retriever 0.16\n",
      "plunger, plumber's helper 0.18\n",
      "water jug 0.18\n",
      "bow tie, bow-tie, bowtie 0.2\n",
      "wooden spoon 0.2\n",
      "drumstick 0.22\n",
      "dumbbell 0.22\n",
      "nail 0.22\n",
      "pop bottle, soda bottle 0.22\n",
      "reel 0.22\n",
      "bucket, pail 0.24\n",
      "hog, pig, grunter, squealer, Sus scrofa 0.24\n",
      "umbrella 0.24\n",
      "barbershop 0.26\n"
     ]
    }
   ],
   "source": [
    "most_inaccurate_k_classes(per_class_acc, 15, decode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### most confused pairs of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tabby, tabby cat', 'Egyptian cat'),\n",
       " ('walking stick, walkingstick, stick insect', 'mantis, mantid'),\n",
       " ('apron', 'poncho'),\n",
       " ('basketball', 'volleyball'),\n",
       " ('beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon',\n",
       "  'convertible'),\n",
       " ('beer bottle', 'pop bottle, soda bottle'),\n",
       " ('convertible', 'sports car, sport car'),\n",
       " ('frying pan, frypan, skillet', 'wok'),\n",
       " ('magnetic compass', 'stopwatch, stop watch'),\n",
       " ('plate', 'ice cream, icecream'),\n",
       " ('mashed potato', 'meat loaf, meatloaf'),\n",
       " ('orange', 'lemon'),\n",
       " ('lakeside, lakeshore', 'seashore, coast, seacoast, sea-coast')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confused_pairs = most_confused_classes(\n",
    "    val_true_targets, val_predictions, decode, min_n_confusions=12\n",
    ")\n",
    "confused_pairs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
